You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/12/18 08:50:47 UTC
tajo git commit: TAJO-1232: Implicit groupby queries with LIMIT lead
to wrong results. (jihoon)
Repository: tajo
Updated Branches:
refs/heads/master 5c852b798 -> af4083bfd
TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/af4083bf
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/af4083bf
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/af4083bf
Branch: refs/heads/master
Commit: af4083bfd337bfd6fe8278b0363c987d19088e50
Parents: 5c852b7
Author: Jihoon Son <ji...@apache.org>
Authored: Thu Dec 18 16:50:09 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Thu Dec 18 16:50:09 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../org/apache/tajo/master/GlobalEngine.java | 8 ++++----
.../tajo/engine/query/TestGroupByQuery.java | 21 ++++++++++++++++++++
.../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 +
.../testGroupbyWithLimit1.result | 3 +++
.../testGroupbyWithLimit2.result | 3 +++
.../testGroupbyWithLimit3.result | 3 +++
.../org/apache/tajo/plan/LogicalPlanner.java | 7 +++++++
10 files changed, 47 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2230d0f..29b66d1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -112,6 +112,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results.
+ (jihoon)
+
TAJO-1254: Fix getProgress race conditions in Query. (jinho)
TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses.
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index a9624f8..71b1f9b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -204,10 +204,10 @@ public class GlobalEngine extends AbstractService {
}
private SubmitQueryResponse executeQueryInternal(QueryContext queryContext,
- Session session,
- LogicalPlan plan,
- String sql,
- String jsonExpr) throws Exception {
+ Session session,
+ LogicalPlan plan,
+ String sql,
+ String jsonExpr) throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index 3dd1219..edbe029 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -749,4 +749,25 @@ public class TestGroupByQuery extends QueryTestCaseBase {
ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal);
}
}
+
+ @Test
+ public final void testGroupbyWithLimit1() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testGroupbyWithLimit2() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testGroupbyWithLimit3() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
new file mode 100644
index 0000000..06d96af
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
@@ -0,0 +1 @@
+select sum(l_linenumber) from lineitem limit 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
new file mode 100644
index 0000000..1ddb171
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
@@ -0,0 +1 @@
+select sum(l_linenumber) from lineitem limit 10;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
new file mode 100644
index 0000000..1bb7c7f
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
@@ -0,0 +1 @@
+select l_orderkey, sum(l_linenumber) from lineitem group by l_orderkey limit 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
new file mode 100644
index 0000000..23bbaea
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
@@ -0,0 +1,3 @@
+?sum
+-------------------------------
+7
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
new file mode 100644
index 0000000..23bbaea
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
@@ -0,0 +1,3 @@
+?sum
+-------------------------------
+7
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
new file mode 100644
index 0000000..cb1e141
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
@@ -0,0 +1,3 @@
+l_orderkey,?sum
+-------------------------------
+3,3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index df45c3a..1a426e0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -716,6 +716,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
LogicalPlan plan = context.plan;
QueryBlock block = context.queryBlock;
+
+ // The limit operation must affect to the number of results, not the number of input records.
+ // Thus, the aggregation must be carried out before the limit operation.
+ if (child.getType() == NodeType.LIMIT) {
+ child = ((LimitNode)child).getChild();
+ }
+
GroupbyNode groupbyNode = context.plan.createNode(GroupbyNode.class);
groupbyNode.setChild(child);
groupbyNode.setInSchema(child.getOutSchema());