You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/02/04 12:14:02 UTC
[1/2] tajo git commit: TAJO-1316: NPE occurs when performing window
functions after join
Repository: tajo
Updated Branches:
refs/heads/branch-0.10.0 953ad6c39 -> 60e6863f8
refs/heads/master ae7862b28 -> 5e024f947
TAJO-1316: NPE occurs when performing window functions after join
Closes #372
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5e024f94
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5e024f94
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5e024f94
Branch: refs/heads/master
Commit: 5e024f947b441fb49f5740deab119867eb655795
Parents: ae7862b
Author: Jihun Kang <ji...@apache.org>
Authored: Wed Feb 4 20:12:16 2015 +0900
Committer: Jihun Kang <ji...@apache.org>
Committed: Wed Feb 4 20:12:16 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../engine/function/TestBuiltinFunctions.java | 38 ++++++++++++++++++++
.../org/apache/tajo/plan/LogicalPlanner.java | 4 +--
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 22 ++++++++++++
4 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5e024f94/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 85d6684..e9a9332 100644
--- a/CHANGES
+++ b/CHANGES
@@ -179,6 +179,9 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1316: NPE occurs when performing window functions after join.
+ (jihun)
+
TAJO-1325: Invalid history cleaner timeout. (jinho)
TAJO-1283: ORDER BY with the first descending order causes wrong results.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5e024f94/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
index 4578ae5..9f68786 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
@@ -352,4 +352,42 @@ public class TestBuiltinFunctions extends QueryTestCaseBase {
assertResultSet(res);
cleanupQuery(res);
}
+
+ @Test
+ public void testRankWithTwoTables() throws Exception {
+ KeyValueSet tableOptions = new KeyValueSet();
+ tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N");
+
+ Schema schema = new Schema();
+ schema.addColumn("id", TajoDataTypes.Type.INT4);
+ String[] data = new String[] {"1", "3", "2", "4"};
+ TajoTestingCluster.createTable("rank_table1", schema, tableOptions, data, 1);
+ schema = new Schema();
+ schema.addColumn("refid", TajoDataTypes.Type.INT4);
+ schema.addColumn("value", TajoDataTypes.Type.TEXT);
+ data = new String[] {"1|efgh", "2|abcd", "4|erjk", "8|dfef"};
+ TajoTestingCluster.createTable("rank_table2", schema, tableOptions, data, 1);
+ ResultSet res = null;
+
+ try {
+ res = executeString("select rank() over (order by id) from rank_table1 a, rank_table2 b "
+ + " where a.id = b.refid");
+ String expectedString = "?windowfunction\n" +
+ "-------------------------------\n" +
+ "1\n" +
+ "2\n" +
+ "3\n";
+
+ assertEquals(expectedString, resultSetToString(res));
+ } finally {
+ if (res != null) {
+ try {
+ res.close();
+ } catch(Throwable ignored) {}
+ }
+ executeString("DROP TABLE rank_table1 PURGE");
+ executeString("DROP TABLE rank_table2 PURGE");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5e024f94/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 9002f28..babcb1e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -1221,7 +1220,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
NamedExpr namedExpr = it.next();
try {
evalNode = exprAnnotator.createEvalNode(context, namedExpr.getExpr(), NameResolvingMode.LEGACY);
- if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) {
+ if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0
+ && EvalTreeUtil.findWindowFunction(evalNode).size() == 0) {
block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode);
newlyEvaluatedExprs.add(namedExpr.getAlias());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5e024f94/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
index f5c2cbd..f1d4498 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
@@ -486,6 +486,28 @@ public class EvalTreeUtil {
return this.aggFucntions;
}
}
+
+ public static Set<WindowFunctionEval> findWindowFunction(EvalNode expr) {
+ AllWindowFunctionFinder finder = new AllWindowFunctionFinder();
+ expr.postOrder(finder);
+ return finder.getWindowFunctionSet();
+ }
+
+ public static class AllWindowFunctionFinder implements EvalNodeVisitor {
+ private Set<WindowFunctionEval> windowFunctions = Sets.newHashSet();
+
+ @Override
+ public void visit(EvalNode node) {
+ if (node.getType() == EvalType.WINDOW_FUNCTION) {
+ WindowFunctionEval field = (WindowFunctionEval) node;
+ windowFunctions.add(field);
+ }
+ }
+
+ public Set<WindowFunctionEval> getWindowFunctionSet() {
+ return windowFunctions;
+ }
+ }
public static <T extends EvalNode> Collection<T> findEvalsByType(EvalNode evalNode, EvalType type) {
EvalFinder finder = new EvalFinder(type);
[2/2] tajo git commit: TAJO-1316: NPE occurs when performing window
functions after join
Posted by ji...@apache.org.
TAJO-1316: NPE occurs when performing window functions after join
Closes #372
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/60e6863f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/60e6863f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/60e6863f
Branch: refs/heads/branch-0.10.0
Commit: 60e6863f8a4657b143b2d7bc864b09f5e1f39021
Parents: 953ad6c
Author: Jihun Kang <ji...@apache.org>
Authored: Wed Feb 4 20:12:16 2015 +0900
Committer: Jihun Kang <ji...@apache.org>
Committed: Wed Feb 4 20:13:20 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../engine/function/TestBuiltinFunctions.java | 38 ++++++++++++++++++++
.../org/apache/tajo/plan/LogicalPlanner.java | 4 +--
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 22 ++++++++++++
4 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/60e6863f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 85d6684..e9a9332 100644
--- a/CHANGES
+++ b/CHANGES
@@ -179,6 +179,9 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1316: NPE occurs when performing window functions after join.
+ (jihun)
+
TAJO-1325: Invalid history cleaner timeout. (jinho)
TAJO-1283: ORDER BY with the first descending order causes wrong results.
http://git-wip-us.apache.org/repos/asf/tajo/blob/60e6863f/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
index 4578ae5..9f68786 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestBuiltinFunctions.java
@@ -352,4 +352,42 @@ public class TestBuiltinFunctions extends QueryTestCaseBase {
assertResultSet(res);
cleanupQuery(res);
}
+
+ @Test
+ public void testRankWithTwoTables() throws Exception {
+ KeyValueSet tableOptions = new KeyValueSet();
+ tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N");
+
+ Schema schema = new Schema();
+ schema.addColumn("id", TajoDataTypes.Type.INT4);
+ String[] data = new String[] {"1", "3", "2", "4"};
+ TajoTestingCluster.createTable("rank_table1", schema, tableOptions, data, 1);
+ schema = new Schema();
+ schema.addColumn("refid", TajoDataTypes.Type.INT4);
+ schema.addColumn("value", TajoDataTypes.Type.TEXT);
+ data = new String[] {"1|efgh", "2|abcd", "4|erjk", "8|dfef"};
+ TajoTestingCluster.createTable("rank_table2", schema, tableOptions, data, 1);
+ ResultSet res = null;
+
+ try {
+ res = executeString("select rank() over (order by id) from rank_table1 a, rank_table2 b "
+ + " where a.id = b.refid");
+ String expectedString = "?windowfunction\n" +
+ "-------------------------------\n" +
+ "1\n" +
+ "2\n" +
+ "3\n";
+
+ assertEquals(expectedString, resultSetToString(res));
+ } finally {
+ if (res != null) {
+ try {
+ res.close();
+ } catch(Throwable ignored) {}
+ }
+ executeString("DROP TABLE rank_table1 PURGE");
+ executeString("DROP TABLE rank_table2 PURGE");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/60e6863f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 9002f28..babcb1e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -1221,7 +1220,8 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
NamedExpr namedExpr = it.next();
try {
evalNode = exprAnnotator.createEvalNode(context, namedExpr.getExpr(), NameResolvingMode.LEGACY);
- if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) {
+ if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0
+ && EvalTreeUtil.findWindowFunction(evalNode).size() == 0) {
block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode);
newlyEvaluatedExprs.add(namedExpr.getAlias());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/60e6863f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
index f5c2cbd..f1d4498 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
@@ -486,6 +486,28 @@ public class EvalTreeUtil {
return this.aggFucntions;
}
}
+
+ public static Set<WindowFunctionEval> findWindowFunction(EvalNode expr) {
+ AllWindowFunctionFinder finder = new AllWindowFunctionFinder();
+ expr.postOrder(finder);
+ return finder.getWindowFunctionSet();
+ }
+
+ public static class AllWindowFunctionFinder implements EvalNodeVisitor {
+ private Set<WindowFunctionEval> windowFunctions = Sets.newHashSet();
+
+ @Override
+ public void visit(EvalNode node) {
+ if (node.getType() == EvalType.WINDOW_FUNCTION) {
+ WindowFunctionEval field = (WindowFunctionEval) node;
+ windowFunctions.add(field);
+ }
+ }
+
+ public Set<WindowFunctionEval> getWindowFunctionSet() {
+ return windowFunctions;
+ }
+ }
public static <T extends EvalNode> Collection<T> findEvalsByType(EvalNode evalNode, EvalType type) {
EvalFinder finder = new EvalFinder(type);