You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2014/02/24 10:42:10 UTC
git commit: TAJO-403: HiveQLAnalyzer should supports standard
function in the GROUP BY Clause. (jaehwa)
Repository: incubator-tajo
Updated Branches:
refs/heads/master ab38b5b46 -> 7a05b2158
TAJO-403: HiveQLAnalyzer should supports standard function in the GROUP BY Clause. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/7a05b215
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/7a05b215
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/7a05b215
Branch: refs/heads/master
Commit: 7a05b21586c7588e1b4fd26bed6b41f1ebccb499
Parents: ab38b5b
Author: blrunner <jh...@gruter.com>
Authored: Mon Feb 24 18:41:58 2014 +0900
Committer: blrunner <jh...@gruter.com>
Committed: Mon Feb 24 18:41:58 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../tajo/engine/parser/HiveQLAnalyzer.java | 88 ++++++++------------
.../tajo/engine/parser/TestHiveQLAnalyzer.java | 31 +++++++
.../resources/queries/default/groupby_2.sql | 1 +
.../resources/queries/default/groupby_3.sql | 1 +
.../resources/queries/default/groupby_4.sql | 1 +
.../resources/queries/default/groupby_5.sql | 1 +
7 files changed, 71 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index baf0e29..27f7698 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -263,6 +263,8 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-403: HiveQLAnalyzer should supports standard function in the GROUP BY Clause. (jaehwa)
+
TAJO-594: MySQL store doesn't work. (Yongjun Park via jaehwa)
TAJO-590: Rename HiveConverter to HiveQLAnalyzer. (jaehwa)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
index 1228324..fbd483c 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java
@@ -30,10 +30,7 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.engine.parser.HiveQLParser.TableAllColumnsContext;
import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> {
private static final Log LOG = LogFactory.getLog(HiveQLAnalyzer.class.getName());
@@ -1191,6 +1188,14 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> {
*/
@Override
public Expr visitCastExpression(HiveQLParser.CastExpressionContext ctx) {
+ LOG.info("### 100 - expr:" + ctx.getText());
+ LOG.info("### 110 - primitiveType:" + ctx.primitiveType().getText());
+
+// current = new CastExpr(current, visitData_type(ctx.cast_target(i).data_type()));
+
+
+ Expr expr = visitExpression(ctx.expression());
+ LOG.info("### 200 - expr:" + expr.toJson());
return visitExpression(ctx.expression());
}
@@ -1259,66 +1264,41 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor<Expr> {
@Override
public Aggregation visitGroupByClause(HiveQLParser.GroupByClauseContext ctx) {
Aggregation clause = new Aggregation();
+
if (ctx.groupByExpression().size() > 0) {
- List<Expr> columns = new ArrayList<Expr>();
- List<Expr> functions = new ArrayList<Expr>();
+ int elementSize = ctx.groupByExpression().size();
+ ArrayList<Aggregation.GroupElement> groups = new ArrayList<Aggregation.GroupElement>(elementSize + 1);
+ ArrayList<Expr> ordinaryExprs = new ArrayList<Expr>();
+ int groupSize = 1;
+ groups.add(null);
for (int i = 0; i < ctx.groupByExpression().size(); i++) {
Expr expr = visitGroupByExpression(ctx.groupByExpression(i));
- if (expr instanceof ColumnReferenceExpr) {
- columns.add(expr);
- } else if (expr instanceof FunctionExpr) {
- functions.add(expr);
- } else {
- //TODO: find another case.
- }
- }
-
- Aggregation.GroupElement[] groups = null;
-
- if (columns.size() > 0) {
- groups = new Aggregation.GroupElement[1 + functions.size()];
- } else {
- groups = new Aggregation.GroupElement[functions.size()];
- }
+ if (expr instanceof FunctionExpr) {
+ FunctionExpr function = (FunctionExpr) expr;
- int index = 0;
- if (columns.size() > 0) {
- index = 0;
- ColumnReferenceExpr[] columnReferenceExprs = new ColumnReferenceExpr[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- ColumnReferenceExpr expr = (ColumnReferenceExpr) columns.get(i);
- columnReferenceExprs[i] = expr;
- }
- groups[index] = new Aggregation.GroupElement(Aggregation.GroupType.OrdinaryGroup, columnReferenceExprs);
- }
-
- if (functions.size() > 0) {
- if (columns.size() == 0) {
- index = 0;
+ if (function.getSignature().equalsIgnoreCase("ROLLUP")) {
+ groupSize++;
+ groups.add(new Aggregation.GroupElement(Aggregation.GroupType.Rollup,
+ function.getParams()));
+ } else if (function.getSignature().equalsIgnoreCase("CUBE")) {
+ groupSize++;
+ groups.add(new Aggregation.GroupElement(Aggregation.GroupType.Cube, function.getParams()));
+ } else {
+ Collections.addAll(ordinaryExprs, function);
+ }
} else {
- index = 1;
- }
-
- for (int i = 0; i < functions.size(); i++) {
- FunctionExpr function = (FunctionExpr) functions.get(i);
-
- Expr[] params = function.getParams();
- ColumnReferenceExpr[] column = new ColumnReferenceExpr[params.length];
- for (int j = 0; j < column.length; j++)
- column[j] = (ColumnReferenceExpr) params[j];
-
- if (function.getSignature().equalsIgnoreCase("ROLLUP"))
- groups[i + index] = new Aggregation.GroupElement(Aggregation.GroupType.Rollup, column);
- else if (function.getSignature().equalsIgnoreCase("CUBE"))
- groups[i + index] = new Aggregation.GroupElement(Aggregation.GroupType.Cube, column);
- else
- throw new RuntimeException("Unexpected aggregation function.");
+ Collections.addAll(ordinaryExprs, (ColumnReferenceExpr)expr);
}
}
- clause.setGroups(groups);
+ if (ordinaryExprs != null) {
+ groups.set(0, new Aggregation.GroupElement(Aggregation.GroupType.OrdinaryGroup, ordinaryExprs.toArray(new Expr[ordinaryExprs.size()])));
+ clause.setGroups(groups.subList(0, groupSize).toArray(new Aggregation.GroupElement[groupSize]));
+ } else if (groupSize > 1) {
+ clause.setGroups(groups.subList(1, groupSize).toArray(new Aggregation.GroupElement[groupSize - 1]));
+ }
}
//TODO: grouping set expression
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
index dcdd862..5d5ad4d 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/parser/TestHiveQLAnalyzer.java
@@ -211,6 +211,37 @@ public class TestHiveQLAnalyzer {
}
@Test
+ public void testGroupby2() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/groupby_2.sql"));
+ Expr expr = parseQuery(sql);
+ Expr hiveExpr = parseHiveQL(sql);
+ compareJsonResult(expr, hiveExpr);
+ }
+
+ @Test
+ public void testGroupby3() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/groupby_3.sql"));
+ Expr expr = parseQuery(sql);
+ Expr hiveExpr = parseHiveQL(sql);
+ compareJsonResult(expr, hiveExpr);
+ }
+
+ @Test
+ public void testGroupby4() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/groupby_4.sql"));
+ Expr expr = parseQuery(sql);
+ Expr hiveExpr = parseHiveQL(sql);
+ compareJsonResult(expr, hiveExpr);
+ }
+
+ @Test
+ public void testGroupby5() throws IOException {
+ String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/groupby_5.sql"));
+ Expr expr = parseQuery(sql);
+ Expr hiveExpr = parseHiveQL(sql);
+ compareJsonResult(expr, hiveExpr);
+ }
+ @Test
public void testJoin2() throws IOException {
String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_2.sql"));
Expr expr = parseQuery(sql);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_2.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_2.sql
new file mode 100644
index 0000000..031cab2
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_2.sql
@@ -0,0 +1 @@
+select col0, col1, col2, col3, sum(col4) as total, avg(col5) from base group by col0, cube (col1, col2), col3 having total > 100
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_3.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_3.sql
new file mode 100644
index 0000000..469e8b2
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_3.sql
@@ -0,0 +1 @@
+select col0, col1, col2, col3, sum(col4) as total, avg(col5) from base group by col0, col3, cube (col1, col2) having total > 100
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_4.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_4.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_4.sql
new file mode 100644
index 0000000..749c6d9
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_4.sql
@@ -0,0 +1 @@
+select trim(name), count(1) from table1 group by trim(name)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/7a05b215/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_5.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_5.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_5.sql
new file mode 100644
index 0000000..f654ef6
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/default/groupby_5.sql
@@ -0,0 +1 @@
+select id, name, count(*), sum(id), avg(age) from people group by id, name
\ No newline at end of file