You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2020/07/07 21:06:39 UTC
[impala] 03/03: IMPALA-9897: GROUPING SETS/CUBE/ROLLUP parsing and
analysis
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7a5b1bab5adde5cdf4cd3f772edc5631efd3737e
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Thu Jun 25 10:32:07 2020 -0700
IMPALA-9897: GROUPING SETS/CUBE/ROLLUP parsing and analysis
Supports a single ROLLUP, CUBE or GROUPING SETS clause in
GROUP BY.
Also adds non-standard "WITH ROLLUP" and "WITH CUBE" syntax
that is supported by some other SQL dialects.
This implements basic parsing and validation of the query,
then raises an AnalysisException to report that it is not
supported so that incorrect plans will not be generated.
This patch adds a GroupByClause to each SelectStmt that
contains info about the grouping sets and the original
GROUP BY list. The grouping exprs are still represented
as a List<Expr> in SelectStmt. Most of the logic, including
statement and expr rewrites, can operate on this list of
expressions without requiring special handling for grouping
sets.
Testing:
* Add Parser test.
* Add toSql() test.
* Added analysis tests to check that analysis accepts or rejects
queries correctly.
Change-Id: I178e45de94d736630c97ae1ec4a92423cd74621f
Reviewed-on: http://gerrit.cloudera.org:8080/16112
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/main/cup/sql-parser.cup | 76 ++++-
.../org/apache/impala/analysis/GroupByClause.java | 314 +++++++++++++++++++++
.../org/apache/impala/analysis/SelectStmt.java | 52 +++-
.../org/apache/impala/analysis/StmtRewriter.java | 6 +-
.../java/org/apache/impala/common/RuntimeEnv.java | 10 +
fe/src/main/jflex/sql-scanner.flex | 10 +-
.../apache/impala/analysis/AnalyzeStmtsTest.java | 215 +++++++++++++-
.../org/apache/impala/analysis/AnalyzerTest.java | 16 ++
.../org/apache/impala/analysis/ParserTest.java | 55 ++++
.../java/org/apache/impala/analysis/ToSqlTest.java | 28 ++
10 files changed, 759 insertions(+), 23 deletions(-)
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 3679b97..d7d1c26 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -284,13 +284,14 @@ terminal
KW_ARRAY, KW_AS, KW_ASC, KW_AUTHORIZATION, KW_AVRO, KW_BETWEEN, KW_BIGINT, KW_BINARY,
KW_BLOCKSIZE, KW_BOOLEAN, KW_BY, KW_CACHED, KW_CASCADE, KW_CASE, KW_CAST, KW_CHANGE,
KW_CHAR, KW_CLASS, KW_CLOSE_FN, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMPRESSION,
- KW_COMPUTE, KW_CONSTRAINT, KW_COPY, KW_CREATE, KW_CROSS, KW_CURRENT, KW_DATA,
+ KW_COMPUTE, KW_CONSTRAINT, KW_COPY, KW_CREATE, KW_CROSS, KW_CUBE, KW_CURRENT, KW_DATA,
KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DECIMAL, KW_DEFAULT, KW_DELETE,
KW_DELIMITED, KW_DESC, KW_DESCRIBE, KW_DISABLE, KW_DISTINCT, KW_DIV, KW_DOUBLE,
KW_DROP, KW_ELSE, KW_ENABLE, KW_ENCODING, KW_END, KW_ESCAPED, KW_EXISTS, KW_EXPLAIN,
KW_EXTENDED, KW_EXTERNAL, KW_FALSE, KW_FIELDS, KW_FILEFORMAT, KW_FILES, KW_FINALIZE_FN,
KW_FIRST, KW_FLOAT, KW_FOLLOWING, KW_FOR, KW_FOREIGN, KW_FORMAT, KW_FORMATTED,
- KW_FROM, KW_FULL, KW_FUNCTION, KW_FUNCTIONS, KW_GRANT, KW_GROUP, KW_HASH, KW_HUDIPARQUET,
+ KW_FROM, KW_FULL, KW_FUNCTION, KW_FUNCTIONS, KW_GRANT, KW_GROUP, KW_GROUPING, KW_HASH,
+ KW_HUDIPARQUET,
KW_IGNORE, KW_HAVING, KW_ICEBERG, KW_IF, KW_ILIKE, KW_IN, KW_INCREMENTAL, KW_INIT_FN, KW_INNER,
KW_INPATH, KW_INSERT, KW_INT, KW_INTERMEDIATE, KW_INTERVAL, KW_INTO, KW_INVALIDATE, KW_IREGEXP,
KW_IS, KW_JOIN, KW_KUDU, KW_LAST, KW_LEFT, KW_LEXICAL, KW_LIKE, KW_LIMIT, KW_LINES,
@@ -300,9 +301,10 @@ terminal
KW_PARTITIONS, KW_PRECEDING, KW_PREPARE_FN, KW_PRIMARY, KW_PRODUCED, KW_PURGE,
KW_RANGE, KW_RCFILE, KW_RECOVER, KW_REFERENCES, KW_REFRESH, KW_REGEXP, KW_RELY,
KW_RENAME, KW_REPEATABLE, KW_REPLACE, KW_REPLICATION, KW_RESTRICT, KW_RETURNS,
- KW_REVOKE, KW_RIGHT, KW_RLIKE, KW_ROLE, KW_ROLES, KW_ROW, KW_ROWS, KW_SCHEMA,
+ KW_REVOKE, KW_RIGHT, KW_RLIKE, KW_ROLE, KW_ROLES, KW_ROLLUP, KW_ROW, KW_ROWS, KW_SCHEMA,
KW_SCHEMAS, KW_SELECT, KW_SEMI, KW_SEQUENCEFILE, KW_SERDEPROPERTIES, KW_SERIALIZE_FN,
- KW_SET, KW_SHOW, KW_SMALLINT, KW_SORT, KW_SPEC, KW_STORED, KW_STRAIGHT_JOIN, KW_STRING,
+ KW_SET, KW_SHOW, KW_SMALLINT, KW_SETS, KW_SORT, KW_SPEC, KW_STORED, KW_STRAIGHT_JOIN,
+ KW_STRING,
KW_STRUCT, KW_SYMBOL, KW_TABLE, KW_TABLES, KW_TABLESAMPLE, KW_TBLPROPERTIES,
KW_TERMINATED, KW_TEXTFILE, KW_THEN, KW_TIMESTAMP, KW_TINYINT, KW_TRUNCATE, KW_STATS,
KW_TO, KW_TRUE, KW_UNBOUNDED, KW_UNCACHED, KW_UNION, KW_UNKNOWN, KW_UPDATE,
@@ -386,7 +388,10 @@ nonterminal Expr where_clause;
nonterminal Expr predicate, bool_test_expr;
nonterminal Predicate between_predicate, comparison_predicate, compound_predicate,
in_predicate, like_predicate, exists_predicate;
-nonterminal List<Expr> group_by_clause, opt_partition_by_clause;
+nonterminal List<Expr> grouping_set;
+nonterminal List<List<Expr>> grouping_sets;
+nonterminal GroupByClause group_by_clause;
+nonterminal List<Expr> opt_partition_by_clause;
nonterminal Expr having_clause;
nonterminal List<OrderByElement> order_by_elements, opt_order_by_clause;
nonterminal OrderByElement order_by_element;
@@ -3063,13 +3068,64 @@ where_clause ::=
{: RESULT = null; :}
;
+// Support the following variants:
+// * GROUP BY a, b, c
+// * GROUP BY ROLLUP(a, b, c) - standard ROLLUP syntax
+// * GROUP BY a, b, c WITH ROLLUP - non-standard ROLLUP syntax supported by some systems
+// * GROUP BY CUBE(a, b, c) - standard CUBE syntax
+// * GROUP BY a, b, c WITH CUBE - non-standard CUBE syntax supported by some systems
+// * GROUP BY GROUPING SETS((a, b, c), (a, b), ()) - standard GROUPING SETs syntax
+//
+// Not yet supported: multiple ROLLUP/CUBE/GROUPING SET clauses, e.g.:
+// * GROUP BY a, ROLLUP(b, c)
group_by_clause ::=
KW_GROUP KW_BY expr_list:l
- {: RESULT = l; :}
+ {: RESULT = new GroupByClause(l, GroupByClause.GroupingSetsType.NONE); :}
+ | KW_GROUP KW_BY expr_list:l KW_WITH KW_ROLLUP
+ {: RESULT = new GroupByClause(l, GroupByClause.GroupingSetsType.ROLLUP); :}
+ | KW_GROUP KW_BY KW_ROLLUP LPAREN expr_list:l RPAREN
+ {: RESULT = new GroupByClause(l, GroupByClause.GroupingSetsType.ROLLUP); :}
+ | KW_GROUP KW_BY KW_CUBE LPAREN expr_list:l RPAREN
+ {: RESULT = new GroupByClause(l, GroupByClause.GroupingSetsType.CUBE); :}
+ | KW_GROUP KW_BY expr_list:l KW_WITH KW_CUBE
+ {: RESULT = new GroupByClause(l, GroupByClause.GroupingSetsType.CUBE); :}
+ | KW_GROUP KW_BY KW_GROUPING KW_SETS LPAREN grouping_sets:sets RPAREN
+ {: RESULT = new GroupByClause(sets); :}
| /* empty */
{: RESULT = null; :}
;
+grouping_set ::=
+ expr:e
+ {:
+ List<Expr> list = new ArrayList<Expr>();
+ list.add(e);
+ RESULT = list;
+ :}
+ | LPAREN expr_list:l RPAREN
+ {:
+ RESULT = l;
+ :}
+ | LPAREN RPAREN
+ {:
+ RESULT = new ArrayList<Expr>();
+ :}
+ ;
+
+grouping_sets ::=
+ grouping_set:s
+ {:
+ List<List<Expr>> sets = new ArrayList<List<Expr>>();
+ sets.add(s);
+ RESULT = sets;
+ :}
+ | grouping_sets:sets COMMA grouping_set:s
+ {:
+ sets.add(s);
+ RESULT = sets;
+ :}
+ ;
+
having_clause ::=
KW_HAVING expr:e
{: RESULT = e; :}
@@ -3771,6 +3827,8 @@ word ::=
{: RESULT = r.toString(); :}
| KW_CROSS:r
{: RESULT = r.toString(); :}
+ | KW_CUBE:r
+ {: RESULT = r.toString(); :}
| KW_CURRENT:r
{: RESULT = r.toString(); :}
| KW_DATA:r
@@ -3859,6 +3917,8 @@ word ::=
{: RESULT = r.toString(); :}
| KW_GROUP:r
{: RESULT = r.toString(); :}
+ | KW_GROUPING:r
+ {: RESULT = r.toString(); :}
| KW_HAVING:r
{: RESULT = r.toString(); :}
| KW_HASH:r
@@ -4009,6 +4069,8 @@ word ::=
{: RESULT = r.toString(); :}
| KW_ROLES:r
{: RESULT = r.toString(); :}
+ | KW_ROLLUP:r
+ {: RESULT = r.toString(); :}
| KW_ROW:r
{: RESULT = r.toString(); :}
| KW_ROWS:r
@@ -4029,6 +4091,8 @@ word ::=
{: RESULT = r.toString(); :}
| KW_SET:r
{: RESULT = r.toString(); :}
+ | KW_SETS:r
+ {: RESULT = r.toString(); :}
| KW_SHOW:r
{: RESULT = r.toString(); :}
| KW_SMALLINT:r
diff --git a/fe/src/main/java/org/apache/impala/analysis/GroupByClause.java b/fe/src/main/java/org/apache/impala/analysis/GroupByClause.java
new file mode 100644
index 0000000..d0276f3
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/analysis/GroupByClause.java
@@ -0,0 +1,314 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.analysis;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.impala.common.AnalysisException;
+import com.google.common.base.Preconditions;
+
+
+/**
+ * Representation of the group by clause within a select statement.
+ */
+public class GroupByClause {
+ private final static Logger LOG = LoggerFactory.getLogger(GroupByClause.class);
+
+ public enum GroupingSetsType {NONE, CUBE, SETS, ROLLUP};
+
+ private final static int GROUPING_SET_LIMIT = 64;
+
+ // Original grouping exprs from the parser. Never analyzed or modified.
+ // Non-null.
+ private final List<Expr> origGroupingExprs_;
+
+ // The type of grouping set. Non-null.
+ private final GroupingSetsType groupingSetsType_;
+
+ // Indices of the expressions in 'origGroupingExprs_' referenced by the GROUPING SETS
+ // clause. Non-null iff groupingSetsType is SETS.
+ private final List<List<Integer>> groupingSetsList_;
+
+ // ID of each of the distinct grouping sets. Each ID is a bitfield with a bit set if
+ // that distinct grouping expr is part of that grouping set. Populated during analysis.
+ private final List<Long> groupingIDs_;
+
+ // The list of analyzed expressions in each distinct grouping set. Populated during
+ // analysis.
+ private final List<List<Expr>> analyzedGroupingSets_;
+
+ /**
+ * Constructor for regular GROUP BY, ROLLUP and CUBE.
+ */
+ GroupByClause(List<Expr> groupingExprs,
+ GroupingSetsType groupingSetsType) {
+ Preconditions.checkNotNull(groupingSetsType);
+ Preconditions.checkState(groupingSetsType != GroupingSetsType.SETS);
+ origGroupingExprs_ = groupingExprs;
+ groupingSetsType_ = groupingSetsType;
+ groupingSetsList_ = null;
+ groupingIDs_ = new ArrayList<>();
+ analyzedGroupingSets_ = new ArrayList<>();
+ }
+
+ /**
+ * Constructor for GROUPING SETS
+ */
+ GroupByClause(List<List<Expr>> groupingSetsList) {
+ groupingSetsType_ = GroupingSetsType.SETS;
+ groupingIDs_ = new ArrayList<Long>();
+ groupingSetsList_ = new ArrayList<>();
+ analyzedGroupingSets_ = new ArrayList<>();
+
+ // Extract unique expressions from the list and clone into 'origGroupingExprs_'.
+ origGroupingExprs_ = new ArrayList<Expr>();
+ for (List<Expr> groupingSetExprs : groupingSetsList) {
+ List<Integer> exprIndices = new ArrayList<>();
+ groupingSetsList_.add(exprIndices);
+ for (Expr e : groupingSetExprs) {
+ int idx = origGroupingExprs_.indexOf(e);
+ if (idx < 0) {
+ idx = origGroupingExprs_.size();
+ origGroupingExprs_.add(e);
+ }
+ exprIndices.add(idx);
+ }
+ }
+ }
+
+ /**
+ * Constructor for unanalyzed clone of GROUPING SETS.
+ */
+ GroupByClause(List<Expr> groupingExprs, List<List<Integer>> groupingSetsList) {
+ groupingSetsType_ = GroupingSetsType.SETS;
+ groupingIDs_ = new ArrayList<Long>();
+ groupingSetsList_ = groupingSetsList;
+ origGroupingExprs_ = groupingExprs;
+ analyzedGroupingSets_ = new ArrayList<>();
+ }
+
+ public List<Expr> getOrigGroupingExprs() { return origGroupingExprs_; }
+ public List<Long> getGroupingIDs() { return groupingIDs_; }
+
+ /**
+ * Add a new grouping ID if it is not already present in 'groupingIDs_'. If it is a new
+ * grouping ID, expand the list of expressions it references and append them to
+ * 'analyzedGroupingSets_'.
+ * @param groupingExprs duplicated list of analyzed grouping exprs
+ */
+ private void addGroupingID(long id, List<Expr> groupingExprs) throws AnalysisException {
+ Preconditions.checkState(id >= 0 && id < (1L << groupingExprs.size()),
+ "bad id: " + id);
+ if (groupingIDs_.contains(id)) return;
+ if (groupingIDs_.size() >= GROUPING_SET_LIMIT) {
+ throw new AnalysisException("Limit of " + GROUPING_SET_LIMIT +
+ " grouping sets exceeded");
+ }
+ groupingIDs_.add(id);
+
+ List<Expr> groupingSet = new ArrayList<>();
+ for (int i = 0; i < groupingExprs.size(); ++i) {
+ Expr groupingExpr = groupingExprs.get(i);
+ if ((id & (1L << i)) != 0) {
+ groupingSet.add(groupingExpr);
+ } else {
+ // Populate NULL slots so that tuple layouts are identical.
+ // TODO: it would probably be more efficient to omit these slots and handle it
+ // in transposition.
+ groupingSet.add(NullLiteral.create(groupingExpr.getType()));
+ }
+ }
+ analyzedGroupingSets_.add(groupingSet);
+ }
+
+ public boolean hasGroupingSets() {
+ Preconditions.checkState(groupingSetsType_ != null);
+ return groupingSetsType_ != GroupingSetsType.NONE;
+ }
+
+ public String getTypeString() {
+ return hasGroupingSets() ? groupingSetsType_.toString() : "";
+ }
+
+ public GroupByClause clone() {
+ List<Expr> groupingExprsClone = Expr.cloneList(origGroupingExprs_);
+ if (groupingSetsType_ == GroupingSetsType.SETS) {
+ // Safe to share 'groupingSetsList_' because it is not mutated.
+ return new GroupByClause(groupingExprsClone, groupingSetsList_);
+ } else {
+ return new GroupByClause(groupingExprsClone, groupingSetsType_);
+ }
+ }
+
+ /**
+ * Analyze the grouping sets and generate grouping IDs for them.
+ * Should not be called if hasGroupingSets() is false.
+ * @param groupingExprs grouping exprs from SelectStmt, which must line up with the
+ * original grouping exprs, but can have additional exprs appended on end. These
+ * grouping exprs must have ordinals and aliases substituted, i.e. equivalent
+ * expressions in the list must compare equals with Expr.equals().
+ */
+ public void analyzeGroupingSets(List<Expr> groupingExprs) throws AnalysisException {
+ Preconditions.checkState(hasGroupingSets());
+ Preconditions.checkState(groupingExprs.size() >= origGroupingExprs_.size());
+ Preconditions.checkState(groupingIDs_.isEmpty());
+ Preconditions.checkState(analyzedGroupingSets_.isEmpty());
+
+ // Number of grouping expressions in the original clause. 'groupingExprs' may
+ // have additional expressions appended at the end as a result of rewriting.
+ int numOrigGroupingExprs = origGroupingExprs_.size();
+
+ // Deduplicate the grouping exprs that may be referenced multiple times in the
+ // original 'groupingExprs' map. Compute a map of each expr index in
+ // 'origGroupingExprs_' to the index of the deduped expr.
+ Map<Integer, Integer> firstEquivExpr = new HashMap<>();
+ List<Expr> dedupedGroupingSetExprs = new ArrayList<>();
+ for (int i = 0; i < numOrigGroupingExprs; ++i) {
+ int j = dedupedGroupingSetExprs.indexOf(groupingExprs.get(i));
+ if (j == -1) {
+ firstEquivExpr.put(i, dedupedGroupingSetExprs.size());
+ dedupedGroupingSetExprs.add(groupingExprs.get(i));
+ } else {
+ firstEquivExpr.put(i, j);
+ }
+ }
+
+ int numGroupingSetExprs = dedupedGroupingSetExprs.size();
+ if (numGroupingSetExprs >= (Long.SIZE - 1)) {
+ throw new AnalysisException(
+ "Number of grouping columns (" + numGroupingSetExprs + ") exceeds " +
+ "GROUP BY with " + getTypeString() + " limit of " + (Long.SIZE - 1));
+ }
+
+ // Generate the grouping IDs. Each grouping ID is a bitfield tracking which exprs
+ // in 'dedupedGroupingSetExprs' are included in that group.
+ if (groupingSetsType_ == GroupingSetsType.CUBE) {
+ // Enumerate all combinations of bits, i.e. all integers [0, numGroupingSetExprs).
+ // E.g. for CUBE(a, b, c), we enumerate 111, 110, 101, 100, 010, 001, 000,
+ // meaning the sets (a, b, c), (b, c), (a, c), (c), (b), (a), ().
+ for (long id = (1L << numGroupingSetExprs) - 1; id >= 0; id--) {
+ addGroupingID(id, dedupedGroupingSetExprs);
+ }
+ } else if (groupingSetsType_ == GroupingSetsType.ROLLUP) {
+ Preconditions.checkState(numGroupingSetExprs > 0);
+ // Start with all bits set, then unset the bits one-by-one to enumerate all the
+ // grouping sets for rollup.
+ // E.g. for ROLLUP(a, b, c), we enumerate 111, 011, 001, 000, meaning the sets
+ // (a, b, c), (a, b), (a), ().
+ long bit = (1L << numGroupingSetExprs);
+ long id = bit - 1;
+ while (bit != 0) {
+ addGroupingID(id, dedupedGroupingSetExprs);
+ bit >>= 1;
+ id &= ~bit;
+ }
+ Preconditions.checkState(id == 0);
+ } else {
+ Preconditions.checkState(groupingSetsType_ == GroupingSetsType.SETS);
+ for (List<Integer> set : groupingSetsList_) {
+ // Construct a mask with a bit set for each expression in the grouping set.
+ // We rely on addGroupingID() to ignore duplicate masks.
+ long mask = 0;
+ for (int origPos : set) {
+ Preconditions.checkState(origPos >= 0, "bad pos" + origPos);
+ Preconditions.checkState(origPos < numOrigGroupingExprs, "bad pos " + origPos);
+ int dedupedPos = firstEquivExpr.get(origPos);
+ Preconditions.checkState(dedupedPos >= 0, "bad pos" + dedupedPos);
+ Preconditions.checkState(dedupedPos < numGroupingSetExprs,
+ "bad pos" + dedupedPos);
+ mask |= (1L << dedupedPos);
+ }
+ addGroupingID(mask, dedupedGroupingSetExprs);
+ }
+ }
+ }
+
+ /**
+ * Reset analysis state from call to analyze().
+ */
+ void reset() {
+ // origGroupingExprs_ not modified during analysis.
+ // groupingIDs_ were generated during analysis, so clear them out.
+ groupingIDs_.clear();
+ analyzedGroupingSets_.clear();
+ }
+
+ /**
+ * Generate SQL for this GROUP BY clause.
+ * Note that if additional grouping exprs were appended during rewriting, the resulting
+ * SQL may not be parseable by Impala.
+ * @param groupingExprs grouping exprs from SelectStmt, which must line up with the
+ * original grouping exprs, but can have additional exprs appended on end.
+ */
+ public String toSql(List<Expr> groupingExprs, ToSqlOptions options) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" GROUP BY ");
+ if (groupingSetsType_ == GroupingSetsType.NONE) {
+ sb.append(Expr.toSql(groupingExprs, options));
+ } else {
+ if (groupingSetsType_ == GroupingSetsType.CUBE) {
+ // Use the standard syntax even if input was non-standard "WITH CUBE".
+ sb.append("CUBE(");
+ sb.append(Expr.toSql(origGroupingExprs_, options));
+ sb.append(")");
+ } else if (groupingSetsType_ == GroupingSetsType.ROLLUP) {
+ // Use the standard syntax even if input was non-standard "WITH ROLLUP".
+ sb.append("ROLLUP(");
+ sb.append(Expr.toSql(origGroupingExprs_, options));
+ sb.append(")");
+ } else {
+ Preconditions.checkState(groupingSetsType_ == GroupingSetsType.SETS);
+ sb.append("GROUPING SETS(");
+ for (int i = 0; i < groupingSetsList_.size(); ++i) {
+ sb.append("(");
+ List<Expr> groupingSetExprs = new ArrayList<>();
+ for (int exprIdx : groupingSetsList_.get(i)) {
+ groupingSetExprs.add(origGroupingExprs_.get(exprIdx));
+ }
+ sb.append(Expr.toSql(groupingSetExprs, options));
+ sb.append(")");
+ sb.append((i < groupingSetsList_.size() - 1) ? ", " : "");
+ }
+ sb.append(")");
+ // Any additional grouping exprs are not part of CUBE, ROLLUP or GROUPING SETS
+ // clause and should be appended afterwards.
+ if (groupingExprs.size() > origGroupingExprs_.size()) {
+ sb.append(", ");
+ sb.append(Expr.toSql(groupingExprs.subList(
+ origGroupingExprs_.size(), groupingExprs.size()), options));
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ public String toString() {
+ return this.toSql(origGroupingExprs_, ToSqlOptions.DEFAULT);
+ }
+
+ public String debugString() {
+ return this.toSql(origGroupingExprs_, ToSqlOptions.SHOW_IMPLICIT_CASTS);
+ }
+}
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index ab4a425..440dd70 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -32,6 +32,7 @@ import org.apache.impala.catalog.StructType;
import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.ColumnAliasGenerator;
+import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.common.TableAliasGenerator;
import org.apache.impala.common.TreeNode;
import org.apache.impala.rewrite.ExprRewriter;
@@ -90,7 +91,17 @@ public class SelectStmt extends QueryStmt {
protected final List<String> colLabels_; // lower case column labels
protected final FromClause fromClause_;
protected Expr whereClause_;
+
+ // Grouping expressions for this select block, including expressions from the original
+ // query statement and additional expressions that may be added during rewriting.
protected List<Expr> groupingExprs_;
+
+ // The original GROUP BY clause with information about grouping sets, etc.
+ // If there was no original GROUP BY clause, but grouping exprs are added during
+ // rewriting, this is initialized as a placeholder empty group by list.
+ // Non-null iff groupingExprs_ is non-null.
+ private GroupByClause groupByClause_;
+
protected Expr havingClause_; // original having clause
// havingClause with aliases and agg output resolved
@@ -115,7 +126,7 @@ public class SelectStmt extends QueryStmt {
SelectStmt(SelectList selectList,
FromClause fromClause,
- Expr wherePredicate, List<Expr> groupingExprs,
+ Expr wherePredicate, GroupByClause groupByClause,
Expr havingPredicate, List<OrderByElement> orderByElements,
LimitElement limitElement) {
super(orderByElements, limitElement);
@@ -126,7 +137,12 @@ public class SelectStmt extends QueryStmt {
fromClause_ = fromClause;
}
whereClause_ = wherePredicate;
- groupingExprs_ = groupingExprs;
+ groupByClause_ = groupByClause;
+ if (groupByClause != null) {
+ groupingExprs_ = Expr.cloneList(groupByClause.getOrigGroupingExprs());
+ } else {
+ groupingExprs_ = null;
+ }
havingClause_ = havingPredicate;
colLabels_ = new ArrayList<>();
havingPred_ = null;
@@ -159,6 +175,18 @@ public class SelectStmt extends QueryStmt {
public boolean hasHavingClause() { return havingClause_ != null; }
public ExprSubstitutionMap getBaseTblSmap() { return baseTblSmap_; }
+ /**
+ * Append additional grouping expressions to the select list. Used by StmtRewriter.
+ */
+ protected void addGroupingExprs(List<Expr> addtlGroupingExprs) {
+ if (groupingExprs_ == null) {
+ groupByClause_ = new GroupByClause(
+ Collections.emptyList(), GroupByClause.GroupingSetsType.NONE);
+ groupingExprs_ = new ArrayList<>();
+ }
+ groupingExprs_.addAll(addtlGroupingExprs);
+ }
+
// Column alias generator used during query rewriting.
private ColumnAliasGenerator columnAliasGenerator_ = null;
public ColumnAliasGenerator getColumnAliasGenerator() {
@@ -724,6 +752,14 @@ public class SelectStmt extends QueryStmt {
}
// initialize groupingExprs_ with the analyzed version
groupingExprs_ = groupingExprsCopy_;
+
+ if (groupByClause_ != null && groupByClause_.hasGroupingSets()) {
+ if (RuntimeEnv.INSTANCE.isGroupingSetsValidationEnabled()) {
+ throw new AnalysisException(groupByClause_.getTypeString() +
+ " not supported in GROUP BY");
+ }
+ groupByClause_.analyzeGroupingSets(groupingExprsCopy_);
+ }
}
private void collectAggExprs() {
@@ -799,6 +835,7 @@ public class SelectStmt extends QueryStmt {
}
Expr.removeDuplicates(aggExprs_);
Expr.removeDuplicates(groupingExprs);
+ // TODO: IMPALA-9898: need to pass in grouping set info for MultiAggregateInfo.
multiAggInfo_ = new MultiAggregateInfo(groupingExprs, aggExprs_);
multiAggInfo_.analyze(analyzer_);
}
@@ -1124,17 +1161,13 @@ public class SelectStmt extends QueryStmt {
strBuilder.append(whereClause_.toSql(options));
}
// Group By clause
- if (groupingExprs_ != null) {
- strBuilder.append(" GROUP BY ");
+ if (groupByClause_ != null) {
// Handle both analyzed (multiAggInfo_ != null) and unanalyzed cases.
// Unanalyzed case us used to generate SQL such as for views.
// See ToSqlUtils.getCreateViewSql().
List<Expr> groupingExprs = multiAggInfo_ == null
? groupingExprs_ : multiAggInfo_.getGroupingExprs();
- for (int i = 0; i < groupingExprs.size(); ++i) {
- strBuilder.append(groupingExprs.get(i).toSql(options));
- strBuilder.append((i+1 != groupingExprs.size()) ? ", " : "");
- }
+ strBuilder.append(groupByClause_.toSql(groupingExprs, options));
}
// Having clause
if (havingClause_ != null) {
@@ -1195,6 +1228,8 @@ public class SelectStmt extends QueryStmt {
whereClause_ = (other.whereClause_ != null) ? other.whereClause_.clone() : null;
groupingExprs_ =
(other.groupingExprs_ != null) ? Expr.cloneList(other.groupingExprs_) : null;
+ groupByClause_ =
+ (other.groupByClause_ != null) ? other.groupByClause_.clone() : null;
havingClause_ = (other.havingClause_ != null) ? other.havingClause_.clone() : null;
colLabels_ = Lists.newArrayList(other.colLabels_);
multiAggInfo_ = (other.multiAggInfo_ != null) ? other.multiAggInfo_.clone() : null;
@@ -1274,6 +1309,7 @@ public class SelectStmt extends QueryStmt {
colLabels_.clear();
fromClause_.reset();
if (whereClause_ != null) whereClause_.reset();
+ if (groupByClause_ != null) groupByClause_.reset();
if (groupingExprs_ != null) Expr.resetList(groupingExprs_);
if (havingClause_ != null) havingClause_.reset();
havingPred_ = null;
diff --git a/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java b/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java
index f98afa4..16d6bf2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java
+++ b/fe/src/main/java/org/apache/impala/analysis/StmtRewriter.java
@@ -815,11 +815,7 @@ public class StmtRewriter {
new SelectList(items, isDistinct, stmt.selectList_.getPlanHints());
// Update subquery's GROUP BY clause
if (groupByExprs != null && !groupByExprs.isEmpty()) {
- if (stmt.hasGroupByClause()) {
- stmt.groupingExprs_.addAll(groupByExprs);
- } else {
- stmt.groupingExprs_ = groupByExprs;
- }
+ stmt.addGroupingExprs(groupByExprs);
}
}
diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
index 53b0150..6c00c82 100644
--- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
+++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
@@ -34,6 +34,9 @@ public class RuntimeEnv {
// service.
private boolean enableMtDopValidation_;
+ // Whether we should allow ROLLUP/CUBE/GROUPING SETS queries to pass analysis
+ private boolean enableGroupingSetsValidation_;
+
public RuntimeEnv() {
reset();
}
@@ -45,6 +48,7 @@ public class RuntimeEnv {
numCores_ = Runtime.getRuntime().availableProcessors();
isTestEnv_ = false;
enableMtDopValidation_ = false;
+ enableGroupingSetsValidation_ = true;
}
public int getNumCores() { return numCores_; }
@@ -53,5 +57,11 @@ public class RuntimeEnv {
public boolean isTestEnv() { return isTestEnv_; }
public boolean isMtDopValidationEnabled() { return enableMtDopValidation_; }
public void setEnableMtDopValidation(boolean v) { enableMtDopValidation_ = v; }
+ public boolean isGroupingSetsValidationEnabled() {
+ return enableGroupingSetsValidation_;
+ }
+ public void setEnableGroupingSetsValidation(boolean v) {
+ enableGroupingSetsValidation_ = v;
+ }
}
diff --git a/fe/src/main/jflex/sql-scanner.flex b/fe/src/main/jflex/sql-scanner.flex
index 4f14de7..7a7e3a5 100644
--- a/fe/src/main/jflex/sql-scanner.flex
+++ b/fe/src/main/jflex/sql-scanner.flex
@@ -84,8 +84,8 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("boolean", SqlParserSymbols.KW_BOOLEAN);
keywordMap.put("by", SqlParserSymbols.KW_BY);
keywordMap.put("cached", SqlParserSymbols.KW_CACHED);
- keywordMap.put("case", SqlParserSymbols.KW_CASE);
keywordMap.put("cascade", SqlParserSymbols.KW_CASCADE);
+ keywordMap.put("case", SqlParserSymbols.KW_CASE);
keywordMap.put("cast", SqlParserSymbols.KW_CAST);
keywordMap.put("change", SqlParserSymbols.KW_CHANGE);
keywordMap.put("char", SqlParserSymbols.KW_CHAR);
@@ -100,6 +100,7 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("copy", SqlParserSymbols.KW_COPY);
keywordMap.put("create", SqlParserSymbols.KW_CREATE);
keywordMap.put("cross", SqlParserSymbols.KW_CROSS);
+ keywordMap.put("cube", new Integer(SqlParserSymbols.KW_CUBE));
keywordMap.put("current", SqlParserSymbols.KW_CURRENT);
keywordMap.put("data", SqlParserSymbols.KW_DATA);
keywordMap.put("database", SqlParserSymbols.KW_DATABASE);
@@ -144,13 +145,14 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("functions", SqlParserSymbols.KW_FUNCTIONS);
keywordMap.put("grant", SqlParserSymbols.KW_GRANT);
keywordMap.put("group", SqlParserSymbols.KW_GROUP);
+ keywordMap.put("grouping", new Integer(SqlParserSymbols.KW_GROUPING));
keywordMap.put("hash", SqlParserSymbols.KW_HASH);
keywordMap.put("having", SqlParserSymbols.KW_HAVING);
keywordMap.put("hudiparquet", SqlParserSymbols.KW_HUDIPARQUET);
keywordMap.put("iceberg", SqlParserSymbols.KW_ICEBERG);
keywordMap.put("if", SqlParserSymbols.KW_IF);
- keywordMap.put("ilike", SqlParserSymbols.KW_ILIKE);
keywordMap.put("ignore", SqlParserSymbols.KW_IGNORE);
+ keywordMap.put("ilike", SqlParserSymbols.KW_ILIKE);
keywordMap.put("in", SqlParserSymbols.KW_IN);
keywordMap.put("incremental", SqlParserSymbols.KW_INCREMENTAL);
keywordMap.put("init_fn", SqlParserSymbols.KW_INIT_FN);
@@ -185,8 +187,8 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("nulls", SqlParserSymbols.KW_NULLS);
keywordMap.put("offset", SqlParserSymbols.KW_OFFSET);
keywordMap.put("on", SqlParserSymbols.KW_ON);
- keywordMap.put("||", SqlParserSymbols.KW_LOGICAL_OR);
keywordMap.put("or", SqlParserSymbols.KW_OR);
+ keywordMap.put("||", SqlParserSymbols.KW_LOGICAL_OR);
keywordMap.put("orc", SqlParserSymbols.KW_ORC);
keywordMap.put("order", SqlParserSymbols.KW_ORDER);
keywordMap.put("outer", SqlParserSymbols.KW_OUTER);
@@ -221,6 +223,7 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("rlike", SqlParserSymbols.KW_RLIKE);
keywordMap.put("role", SqlParserSymbols.KW_ROLE);
keywordMap.put("roles", SqlParserSymbols.KW_ROLES);
+ keywordMap.put("rollup", new Integer(SqlParserSymbols.KW_ROLLUP));
keywordMap.put("row", SqlParserSymbols.KW_ROW);
keywordMap.put("rows", SqlParserSymbols.KW_ROWS);
keywordMap.put("schema", SqlParserSymbols.KW_SCHEMA);
@@ -231,6 +234,7 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("serdeproperties", SqlParserSymbols.KW_SERDEPROPERTIES);
keywordMap.put("serialize_fn", SqlParserSymbols.KW_SERIALIZE_FN);
keywordMap.put("set", SqlParserSymbols.KW_SET);
+ keywordMap.put("sets", new Integer(SqlParserSymbols.KW_SETS));
keywordMap.put("show", SqlParserSymbols.KW_SHOW);
keywordMap.put("smallint", SqlParserSymbols.KW_SMALLINT);
keywordMap.put("sort", SqlParserSymbols.KW_SORT);
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 276840a..9c5f21e 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -2281,6 +2281,219 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
ScalarType.STRING);
}
+ /**
+ * Test ROLLUP, CUBE and GROUPING SETS functionality.
+ */
+ @Test
+ public void TestGroupingSets() throws AnalysisException {
+ RuntimeEnv.INSTANCE.setEnableGroupingSetsValidation(false);
+ // Basic examples of each clause.
+ AnalyzesOk("select count(*) from functional.alltypes " +
+ "group by rollup(int_col, string_col)");
+ AnalyzesOk("select count(*) from functional.alltypes " +
+ "group by int_col, string_col with rollup");
+ AnalyzesOk("select count(*) from functional.alltypes " +
+ "group by int_col, string_col with cube");
+ AnalyzesOk("select count(*) from functional.alltypes " +
+ "group by GROUPING SETS((int_col), (string_col))");
+
+ // Test limits for number of distinct expressions
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= 25; ++i) {
+ sb.append("bool_col" + i + ", tinyint_col" + i + ", smallint_col" + i +
+ ", int_col" + i + ", bigint_col" + i + ", float_col" + i +
+ ", double_col" + i + ", string_col" + i);
+ if (i != 25) {
+ sb.append(",");
+ }
+ }
+ String longColList = sb.toString();
+ AnalysisError("select count(*) from functional.widetable_250_cols " +
+ "group by rollup(" + longColList + ")",
+ "Number of grouping columns (200) exceeds GROUP BY with ROLLUP limit of 63");
+ AnalysisError("select count(*) from functional.widetable_250_cols " +
+ "group by cube(" + longColList + ")",
+ "Number of grouping columns (200) exceeds GROUP BY with CUBE limit of 63");
+ AnalysisError("select count(*) from functional.widetable_250_cols " +
+ "group by grouping sets((" + longColList + "))",
+ "Number of grouping columns (200) exceeds GROUP BY with SETS limit of 63");
+
+ // Duplicating the same column shouldn't increase distinct column count.
+ AnalysisError("select count(*) from functional.widetable_250_cols " +
+ "group by grouping sets((" + longColList + ", string_col1), " +
+ "(bool_col1, int_col1))",
+ "Number of grouping columns (200) exceeds GROUP BY with SETS limit of 63");
+
+ // Grouping sets with overlapping expression lists doesn't hit limit, since exprs are
+ // deduplicated.
+ AnalyzesOk("select count(*) from functional.widetable_250_cols " +
+ "group by grouping sets(( " +
+ " bool_col1, tinyint_col1, smallint_col1, int_col1, bigint_col1, " +
+ " float_col1, double_col1, string_col1, " +
+ " bool_col2, tinyint_col2, smallint_col2, int_col2, bigint_col2, " +
+ " float_col2, double_col2, string_col2, " +
+ " bool_col3, tinyint_col3, smallint_col3, int_col3, bigint_col3, " +
+ " float_col3, double_col3, string_col3, " +
+ " bool_col4, tinyint_col4, smallint_col4, int_col4, bigint_col4, " +
+ " float_col4, double_col4, string_col4, " +
+ " bool_col5, tinyint_col5, smallint_col5, int_col5, bigint_col5, " +
+ " float_col5, double_col5, string_col5, " +
+ " bool_col6, tinyint_col6, smallint_col6, int_col6, bigint_col6, " +
+ " float_col6, double_col6, string_col6, " +
+ " bool_col7, tinyint_col7, smallint_col7, int_col7, bigint_col7, " +
+ " float_col7, double_col7, string_col7), " +
+ " (bool_col1, tinyint_col1, smallint_col1, int_col1, bigint_col1, " +
+ " float_col1, double_col1, string_col1, " +
+ " bool_col2, tinyint_col2, smallint_col2, int_col2, bigint_col2, " +
+ " float_col2, double_col2, string_col2)); ");
+
+ // Test limit of number of distinct grouping sets - 63. CUBE with 6 grouping exprs
+ // results in 2^6 = 64 distinct grouping sets. CUBE with 5 grouping exprs and ROLLUP
+ // with 6 grouping exprs are both under the limit.
+ AnalyzesOk("select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, " +
+ "float_col, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, " +
+ "float_col)");
+ AnalyzesOk("select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, " +
+ "count(*) " +
+ "from functional.alltypes " +
+ "group by cube(id, bool_col, tinyint_col, smallint_col, int_col, bigint_col)");
+ AnalysisError("select id, bool_col, tinyint_col, smallint_col, int_col, " +
+ "bigint_col, float_col, count(*) " +
+ "from functional.alltypes " +
+ "group by cube(id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, " +
+ "float_col)",
+ "Limit of 64 grouping sets exceeded");
+
+
+ // Basic test cases where grouping exprs are subset and superset of select list.
+ AnalyzesOk("select int_col, count(*) from functional.alltypes " +
+ "group by rollup(int_col, string_col)");
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by rollup(int_col, string_col)");
+ AnalysisError("select int_col, bool_col, string_col, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(int_col, string_col)",
+ "select list expression not produced by aggregation output " +
+ "(missing from GROUP BY clause?): bool_col");
+ AnalyzesOk("select int_col, count(*) from functional.alltypes " +
+ "group by cube(int_col, string_col)");
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by cube(int_col, string_col)");
+ AnalysisError("select int_col, bool_col, string_col, count(*) " +
+ "from functional.alltypes " +
+ "group by cube(int_col, string_col)",
+ "select list expression not produced by aggregation output " +
+ "(missing from GROUP BY clause?): bool_col");
+ AnalyzesOk("select int_col, count(*) from functional.alltypes " +
+ "group by grouping sets((int_col, string_col), (int_col))");
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by grouping sets((int_col, string_col), (int_col))");
+ AnalysisError("select int_col, bool_col, string_col, count(*) " +
+ "from functional.alltypes " +
+ "group by grouping sets((int_col, string_col), (int_col))",
+ "select list expression not produced by aggregation output " +
+ "(missing from GROUP BY clause?): bool_col");
+
+ // Support for ordinals.
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by rollup(1, 2)");
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by cube(1, 2)");
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by grouping sets((1, 2), (1))");
+ AnalysisError("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by rollup(1, 3)",
+ "GROUP BY expression must not contain aggregate functions: 3");
+ AnalysisError("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by cube(1, 3)",
+ "GROUP BY expression must not contain aggregate functions: 3");
+ AnalysisError("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by grouping sets((1, 3), (1))",
+ "GROUP BY expression must not contain aggregate functions: 3");
+
+ // Refer to same column by name and ordinal in grouping sets.
+ AnalyzesOk("select int_col, string_col, count(*) from functional.alltypes " +
+ "group by grouping sets((1, 2), (int_col))");
+
+ // Group by non-trivial expressions not in select list
+ AnalyzesOk("select count(*) from functional.alltypes " +
+ "group by rollup(int_col, substring(string_col, 1, 2))");
+ AnalyzesOk("select int_col, count(*) from functional.alltypes " +
+ "group by cube(int_col, substring(string_col, 1, 2))");
+ AnalyzesOk("select int_col, count(*) from functional.alltypes " +
+ "group by grouping sets((int_col), (int_col, substring(string_col, 1, 2)))");
+
+ // Group by non-trivial expressions in select list
+ AnalyzesOk("select int_col, substring(string_col, 1, 2) str, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(int_col, substring(string_col, 1, 2))");
+ AnalyzesOk("select int_col, substring(string_col, 1, 2) str, count(*) " +
+ "from functional.alltypes " +
+ "group by cube(int_col, str)");
+ AnalyzesOk("select int_col, substring(string_col, 1, 2) str, count(*) " +
+ "from functional.alltypes " +
+ "group by grouping sets((int_col, str), (substring(string_col, 1, 2)))");
+
+ // Expressions in select list must appear in group by.
+ AnalysisError("select int_col, substring(string_col, 1, 3) str, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(int_col, substring(string_col, 1, 2))",
+ "select list expression not produced by aggregation output (missing from " +
+ "GROUP BY clause?): substring(string_col, 1, 3)");
+
+ // Use the expression and an alias.
+ AnalyzesOk("select int_col, substring(string_col, 1, 2) str, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(int_col, str, substring(string_col, 1, 2))");
+
+ // Group by constant expression.
+ AnalyzesOk("select int_col, count(*) " +
+ "from functional.alltypes " +
+ "group by rollup(int_col, 'constant')");
+ AnalyzesOk("select int_col, count(*) " +
+ "from functional.alltypes " +
+ "group by cube(int_col, 'constant')");
+ AnalyzesOk("select int_col, count(*) " +
+ "from functional.alltypes " +
+ "group by grouping sets((false), (int_col, 'constant'))");
+
+ // Subqueries with grouping sets inside and outside subquery.
+ AnalyzesOk("select g.int_col, count(*) " +
+ "from functional.alltypesagg g " +
+ "left outer join functional.alltypes a on g.id = a.id " +
+ "where g.int_col < 100 and g.tinyint_col < ( " +
+ " select count(*) from functional.alltypes t " +
+ " where t.id = g.id and g.string_col = t.string_col and t.bool_col = true) " +
+ "group by rollup(g.int_col, g.string_col) " +
+ "having count(*) < 100");
+ AnalysisError("select g.int_col, count(*) " +
+ "from functional.alltypesagg g " +
+ "where g.int_col < 100 and g.tinyint_col < ( " +
+ " select count(*) from functional.alltypes t " +
+ " where t.id = g.id and g.string_col = t.string_col and t.bool_col = true " +
+ " group by rollup(t.string_col, t.bool_col)) " +
+ " group by g.int_col",
+ "Unsupported correlated subquery with grouping and/or aggregation");
+ }
+
+ /**
+ * Test that ROLLUP, CUBE and GROUPING SETS result in AnalysException for now.
+ */
+ @Test
+ public void TestGroupingSetsValidation() throws AnalysisException {
+ AnalysisError("select count(*) from functional.alltypes " +
+ "group by rollup(int_col, string_col)",
+ "ROLLUP not supported in GROUP BY");
+ AnalysisError("select count(*) from functional.alltypes " +
+ "group by cube(int_col, string_col)",
+ "CUBE not supported in GROUP BY");
+ AnalysisError("select count(*) from functional.alltypes " +
+ "group by GROUPING SETS((int_col), (string_col))",
+ "SETS not supported in GROUP BY");
+ }
+
@Test
public void TestDistinct() throws AnalysisException {
AnalyzesOk("select count(distinct id) as sum_id from functional.testtbl");
@@ -4340,4 +4553,4 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
+ "or they should both return 'STRING' or 'VARCHAR' or 'CHAR' types, "
+ "but they return types 'NULL_TYPE' and 'STRING'.");
}
-}
\ No newline at end of file
+}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index 97ea64a..46bc12b 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -28,10 +28,13 @@ import org.apache.impala.catalog.ScalarType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.util.FunctionUtils;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Assume;
+import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@@ -64,6 +67,19 @@ public class AnalyzerTest extends FrontendTestBase {
typeToLiteralValue_.put(Type.NULL, "NULL");
}
+ @Before
+ public void setUpTest() throws Exception {
+ // Reset the RuntimeEnv - individual tests may change it.
+ RuntimeEnv.INSTANCE.reset();
+ RuntimeEnv.INSTANCE.setTestEnv(true);
+ }
+
+ @AfterClass
+ public static void cleanUpClass() throws Exception {
+ // Reset the RuntimeEnv - individual tests may have changed it.
+ RuntimeEnv.INSTANCE.reset();
+ }
+
/**
* Generates and analyzes two variants of the given query by replacing all occurrences
* of "$TBL" in the query string with the unqualified and fully-qualified version of
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 86d4c0b..02b5822 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -3658,6 +3658,61 @@ public class ParserTest extends FrontendTestBase {
}
@Test
+ public void TestRollup() {
+ ParsesOk("SELECT a FROM foo GROUP BY ROLLUP(a)");
+ ParsesOk("SELECT a, b FROM foo GROUP BY ROLLUP(a, b)");
+
+ // Non-standard WITH ROLLUP
+ ParsesOk("SELECT a FROM foo GROUP BY a WITH ROLLUP");
+ ParsesOk("SELECT a, b FROM foo GROUP BY a, b WITH ROLLUP");
+
+ // Nested grouping clauses not supported.
+ ParserError("SELECT a, b FROM foo GROUP BY ROLLUP(a, ROLLUP(b, c))");
+ ParserError("SELECT a, b FROM foo GROUP BY ROLLUP(a, CUBE(b, c))");
+
+ // Multiple clauses not supported with ROLLUP - parser does not handle yet.
+ ParserError("SELECT a, b FROM foo GROUP BY c, ROLLUP(a, b)");
+ ParserError("SELECT a, b FROM foo GROUP BY ROLLUP(a, b), c");
+ ParserError("SELECT a, b FROM foo GROUP BY ROLLUP(a, b), ROLLUP(c)");
+ ParserError("SELECT a, b FROM foo GROUP BY ROLLUP(a, b), CUBE(c, d)");
+ }
+
+ @Test
+ public void TestCube() {
+ ParsesOk("SELECT a FROM foo GROUP BY CUBE(a)");
+ ParsesOk("SELECT a, b FROM foo GROUP BY CUBE(a, b)");
+
+ // Non-standard WITH CUBE is supported
+ ParsesOk("SELECT a FROM foo GROUP BY a WITH CUBE");
+ ParsesOk("SELECT a, b FROM foo GROUP BY a, b WITH CUBE");
+
+ // Nested grouping clauses not supported.
+ ParserError("SELECT a, b FROM foo GROUP BY CUBE(a, ROLLUP(b, c))");
+ ParserError("SELECT a, b FROM foo GROUP BY CUBE(a, CUBE(b, c))");
+
+ // Multiple clauses not supported with CUBE - parser does not handle yet.
+ ParserError("SELECT a, b FROM foo GROUP BY c, CUBE(a, b)");
+ ParserError("SELECT a, b FROM foo GROUP BY CUBE(a, b), c");
+ ParserError("SELECT a, b FROM foo GROUP BY CUBE(a, b), CUBE(c)");
+ }
+
+ @Test
+ public void TestGroupingSets() {
+ ParsesOk("SELECT a FROM foo GROUP BY GROUPING SETS((a, b))");
+ ParsesOk("SELECT a FROM foo GROUP BY GROUPING SETS((a, b), (a), ())");
+
+ // Nested grouping sets not supported.
+ ParserError("SELECT a FROM foo GROUP BY GROUPING SETS(ROLLUP(a, b), (a), ())");
+ ParserError("SELECT a FROM foo GROUP BY GROUPING SETS((a, b), CUBE(a), ())");
+ ParserError("SELECT a FROM foo " +
+ "GROUP BY GROUPING SETS((a, b), GROUPING SETS ((a, b, c), (b, c)))");
+
+ // Multiple clauses not supported with GROUPING SETS - parser does not handle yet.
+ ParserError("SELECT a FROM foo GROUP BY a, b, GROUPING SETS(a, b)");
+ ParserError("SELECT a FROM foo GROUP BY CUBE(a, b), GROUPING SETS(a, b)");
+ }
+
+ @Test
public void TestSet() {
ParsesOk("SET foo='bar'");
ParsesOk("SET foo=\"bar\"");
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index 473808a..4102c7d 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.testutil.TestUtils;
import org.junit.Test;
@@ -814,6 +815,33 @@ public class ToSqlTest extends FrontendTestBase {
"having avg(tinyint_col) > 10 AND count(tinyint_col) > 5",
"SELECT sum(id) FROM functional.alltypes GROUP BY tinyint_col " +
"HAVING avg(tinyint_col) > 10 AND count(tinyint_col) > 5");
+
+ // CUBE, ROLLUP.
+ // Temporarily disable validation of support for CUBE/ROLLUP/GROUPING SETS
+ // so we can run toSql() on these statements.
+ RuntimeEnv.INSTANCE.setEnableGroupingSetsValidation(false);
+ testToSql("select int_col, string_col, sum(id) from functional.alltypes " +
+ "group by rollup(int_col, string_col)",
+ "SELECT int_col, string_col, sum(id) FROM functional.alltypes " +
+ "GROUP BY ROLLUP(int_col, string_col)");
+ testToSql("select int_col, string_col, sum(id) from functional.alltypes " +
+ "group by int_col, string_col with rollup",
+ "SELECT int_col, string_col, sum(id) FROM functional.alltypes " +
+ "GROUP BY ROLLUP(int_col, string_col)");
+ testToSql("select int_col, string_col, sum(id) from functional.alltypes " +
+ "group by cube(int_col, string_col)",
+ "SELECT int_col, string_col, sum(id) FROM functional.alltypes " +
+ "GROUP BY CUBE(int_col, string_col)");
+ testToSql("select int_col, string_col, sum(id) from functional.alltypes " +
+ "group by int_col, string_col with cube",
+ "SELECT int_col, string_col, sum(id) FROM functional.alltypes " +
+ "GROUP BY CUBE(int_col, string_col)");
+
+ // GROUPING SETS.
+ testToSql("select int_col, string_col, sum(id) from functional.alltypes " +
+ "group by grouping sets((int_col, string_col), (int_col), ())",
+ "SELECT int_col, string_col, sum(id) FROM functional.alltypes " +
+ "GROUP BY GROUPING SETS((int_col, string_col), (int_col), ())");
}
// Test the toSql() output of the order by clause.