You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/07/09 06:10:52 UTC
[35/50] [abbrv] git commit: TAJO-884: complex join conditions should
be supported in ON clause.
TAJO-884: complex join conditions should be supported in ON clause.
Closes #44
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b3758361
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b3758361
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b3758361
Branch: refs/heads/window_function
Commit: b3758361335a4cd26d8b7b9d18879b2b35f612fb
Parents: d7c2c95
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Jun 26 11:41:21 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Jun 26 11:41:21 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../apache/tajo/engine/eval/EvalTreeUtil.java | 9 +-
.../tajo/engine/planner/ExprAnnotator.java | 2 +-
.../tajo/engine/planner/ExprNormalizer.java | 44 ++++++--
.../apache/tajo/engine/planner/LogicalPlan.java | 36 ++++---
.../tajo/engine/planner/LogicalPlanner.java | 7 +-
.../engine/planner/logical/join/JoinGraph.java | 25 ++++-
.../planner/rewrite/ProjectionPushDownRule.java | 108 +++++++++++++++----
.../tajo/engine/query/TestCreateTable.java | 2 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 48 +++++++++
.../testLeftOuterJoinWithConstantExpr3.sql | 2 +-
.../queries/TestJoinQuery/table1_int4_ddl.sql | 3 +
.../queries/TestJoinQuery/table1_int8_ddl.sql | 3 +
.../TestJoinQuery/testComplexJoinCondition1.sql | 6 ++
.../TestJoinQuery/testComplexJoinCondition2.sql | 6 ++
.../TestJoinQuery/testComplexJoinCondition3.sql | 6 ++
.../TestJoinQuery/testComplexJoinCondition4.sql | 6 ++
.../testDifferentTypesJoinCondition.sql | 1 +
.../testLeftOuterJoinWithConstantExpr3.sql | 2 +-
.../testComplexJoinCondition1.result | 27 +++++
.../testComplexJoinCondition2.result | 27 +++++
.../testComplexJoinCondition3.result | 27 +++++
.../testComplexJoinCondition4.result | 29 +++++
.../testDifferentTypesJoinCondition.result | 7 ++
24 files changed, 379 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0185044..77186e6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -74,6 +74,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-884: complex join conditions should be supported in ON clause.
+ (hyunsik)
+
TAJO-874: Sometimes InvalidOperationException occurs when aggregates
TableStat. (Hyoungjun Kim via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java
index 07f71dc..3921a7d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java
@@ -248,14 +248,17 @@ public class EvalTreeUtil {
}
BinaryEval binaryEval = (BinaryEval) expr;
- boolean isBothTermFields =
- binaryEval.getLeftExpr().getType() == EvalType.FIELD &&
- binaryEval.getRightExpr().getType() == EvalType.FIELD;
+ boolean isBothTermFields = isSingleColumn(binaryEval.getLeftExpr()) && isSingleColumn(binaryEval.getRightExpr());
+
return joinComparator && isBothTermFields;
} else {
return false;
}
}
+
+ static boolean isSingleColumn(EvalNode evalNode) {
+ return EvalTreeUtil.findUniqueColumns(evalNode).size() == 1;
+ }
public static class ChangeColumnRefVisitor implements EvalNodeVisitor {
private final String findColumn;
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
index 3a53cd2..e143823 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java
@@ -152,7 +152,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
private static EvalNode convertType(EvalNode evalNode, DataType toType) {
// if original and toType is the same, we don't need type conversion.
- if (evalNode.getValueType() == toType) {
+ if (evalNode.getValueType().equals(toType)) {
return evalNode;
}
// the conversion to null is not allowed.
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
index b87665a..75b2b95 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java
@@ -20,6 +20,7 @@ package org.apache.tajo.engine.planner;
import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.engine.exception.NoSuchColumnException;
import java.util.ArrayList;
import java.util.List;
@@ -80,15 +81,21 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR
public static class ExprNormalizedResult {
private final LogicalPlan plan;
private final LogicalPlan.QueryBlock block;
+ private final boolean tryBinaryCommonTermsElimination;
Expr baseExpr; // outmost expressions, which can includes one or more references of the results of aggregation
// function.
List<NamedExpr> aggExprs = new ArrayList<NamedExpr>(); // aggregation functions
List<NamedExpr> scalarExprs = new ArrayList<NamedExpr>(); // scalar expressions which can be referred
- private ExprNormalizedResult(LogicalPlanner.PlanContext context) {
+ private ExprNormalizedResult(LogicalPlanner.PlanContext context, boolean tryBinaryCommonTermsElimination) {
this.plan = context.plan;
this.block = context.queryBlock;
+ this.tryBinaryCommonTermsElimination = tryBinaryCommonTermsElimination;
+ }
+
+ public boolean isBinaryCommonTermsElimination() {
+ return tryBinaryCommonTermsElimination;
}
@Override
@@ -98,7 +105,11 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR
}
public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr expr) throws PlanningException {
- ExprNormalizedResult exprNormalizedResult = new ExprNormalizedResult(context);
+ return normalize(context, expr, false);
+ }
+ public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr expr, boolean subexprElimination)
+ throws PlanningException {
+ ExprNormalizedResult exprNormalizedResult = new ExprNormalizedResult(context, subexprElimination);
Stack<Expr> stack = new Stack<Expr>();
stack.push(expr);
visit(exprNormalizedResult, new Stack<Expr>(), expr);
@@ -152,9 +163,27 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR
return expr;
}
+ private boolean isBinaryCommonTermsElimination(ExprNormalizedResult ctx, Expr expr) {
+ return ctx.isBinaryCommonTermsElimination() && expr.getType() != OpType.Column
+ && ctx.block.namedExprsMgr.contains(expr);
+ }
+
@Override
public Expr visitBinaryOperator(ExprNormalizedResult ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException {
- super.visitBinaryOperator(ctx, stack, expr);
+ stack.push(expr);
+
+ visit(ctx, new Stack<Expr>(), expr.getLeft());
+ if (isBinaryCommonTermsElimination(ctx, expr.getLeft())) {
+ String refName = ctx.block.namedExprsMgr.addExpr(expr.getLeft());
+ expr.setLeft(new ColumnReferenceExpr(refName));
+ }
+
+ visit(ctx, new Stack<Expr>(), expr.getRight());
+ if (isBinaryCommonTermsElimination(ctx, expr.getRight())) {
+ String refName = ctx.block.namedExprsMgr.addExpr(expr.getRight());
+ expr.setRight(new ColumnReferenceExpr(refName));
+ }
+ stack.pop();
////////////////////////
// For Left Term
@@ -249,9 +278,12 @@ class ExprNormalizer extends SimpleAlgebraVisitor<ExprNormalizer.ExprNormalizedR
throws PlanningException {
// if a column reference is not qualified, it finds and sets the qualified column name.
if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) {
- if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName())) {
- String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr);
- expr.setName(normalized);
+ if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) {
+ try {
+ String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr);
+ expr.setName(normalized);
+ } catch (NoSuchColumnException nsc) {
+ }
}
}
return expr;
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
index 0508bac..92df760 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.engine.eval.EvalNode;
+import org.apache.tajo.engine.exception.AmbiguousFieldException;
import org.apache.tajo.engine.exception.NoSuchColumnException;
import org.apache.tajo.engine.exception.VerifyException;
import org.apache.tajo.engine.planner.graph.DirectedGraphCursor;
@@ -384,6 +385,21 @@ public class LogicalPlan {
private Column resolveColumnWithoutQualifier(QueryBlock block,
ColumnReferenceExpr columnRef)throws PlanningException {
+
+ List<Column> candidates = TUtil.newList();
+
+ // It tries to find a full qualified column name from all relations in the current block.
+ for (RelationNode rel : block.getRelations()) {
+ Column found = rel.getTableSchema().getColumn(columnRef.getName());
+ if (found != null) {
+ candidates.add(found);
+ }
+ }
+
+ if (!candidates.isEmpty()) {
+ return ensureUniqueColumn(candidates);
+ }
+
// Trying to find the column within the current block
if (block.currentNode != null && block.currentNode.getInSchema() != null) {
Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName());
@@ -399,7 +415,7 @@ public class LogicalPlan {
}
}
- List<Column> candidates = TUtil.newList();
+
// Trying to find columns from aliased references.
if (block.namedExprsMgr.isAliased(columnRef.getCanonicalName())) {
String originalName = block.namedExprsMgr.getAlias(columnRef.getCanonicalName());
@@ -412,22 +428,10 @@ public class LogicalPlan {
return ensureUniqueColumn(candidates);
}
- // Trying to find columns from other relations in the current block
- for (RelationNode rel : block.getRelations()) {
- Column found = rel.getTableSchema().getColumn(columnRef.getName());
- if (found != null) {
- candidates.add(found);
- }
- }
-
- if (!candidates.isEmpty()) {
- return ensureUniqueColumn(candidates);
- }
-
// This is an exception case. It means that there are some bugs in other parts.
LogicalNode blockRootNode = block.getRoot();
if (blockRootNode != null && blockRootNode.getOutSchema().getColumn(columnRef.getCanonicalName()) != null) {
- throw new VerifyException("ERROR: no such a column name "+ columnRef.getCanonicalName());
+ throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName());
}
// Trying to find columns from other relations in other blocks
@@ -456,7 +460,7 @@ public class LogicalPlan {
return ensureUniqueColumn(candidates);
}
- throw new VerifyException("ERROR: no such a column name "+ columnRef.getCanonicalName());
+ throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName());
}
private static Column ensureUniqueColumn(List<Column> candidates)
@@ -474,7 +478,7 @@ public class LogicalPlan {
}
sb.append(column);
}
- throw new VerifyException("Ambiguous Column Name: " + sb.toString());
+ throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString());
} else {
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index c9948d8..be7bce6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -767,7 +767,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
QueryBlock block = context.queryBlock;
if (join.hasQual()) {
- ExprNormalizedResult normalizedResult = normalizer.normalize(context, join.getQual());
+ ExprNormalizedResult normalizedResult = normalizer.normalize(context, join.getQual(), true);
block.namedExprsMgr.addExpr(normalizedResult.baseExpr);
if (normalizedResult.aggExprs.size() > 0 || normalizedResult.scalarExprs.size() > 0) {
throw new VerifyException("Filter condition cannot include aggregation function");
@@ -1207,13 +1207,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// See PreLogicalPlanVerifier.visitInsert.
// It guarantees that the equivalence between the numbers of target and projected columns.
- ScanNode scanNode = context.plan.createNode(ScanNode.class);
- scanNode.init(desc);
- context.queryBlock.addRelation(scanNode);
String [] targets = expr.getTargetColumns();
Schema targetColumns = new Schema();
for (int i = 0; i < targets.length; i++) {
- Column targetColumn = context.plan.resolveColumn(context.queryBlock, new ColumnReferenceExpr(targets[i]));
+ Column targetColumn = desc.getLogicalSchema().getColumn(targets[i]);
targetColumns.addColumn(targetColumn);
}
insertNode.setTargetSchema(targetColumns);
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java
index 6e321f4..6390a77 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java
@@ -32,6 +32,7 @@ import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.PlanningException;
import org.apache.tajo.engine.planner.graph.SimpleUndirectedGraph;
import org.apache.tajo.engine.planner.logical.JoinNode;
+import org.apache.tajo.engine.planner.logical.RelationNode;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -59,7 +60,17 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> {
String qualifier = CatalogUtil.extractQualifier(columnName);
relationNames[0] = qualifier;
} else {
- throw new PlanningException("Cannot expect a referenced relation: " + leftExpr);
+ // search for a relation which evaluates a right term included in a join condition
+ for (RelationNode rel : block.getRelations()) {
+ if (rel.getOutSchema().contains(leftExpr)) {
+ String qualifier = rel.getCanonicalName();
+ relationNames[0] = qualifier;
+ }
+ }
+
+ if (relationNames[0] == null) { // if not found
+ throw new PlanningException("Cannot expect a referenced relation: " + leftExpr);
+ }
}
}
@@ -71,7 +82,17 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> {
String qualifier = CatalogUtil.extractQualifier(columnName);
relationNames[1] = qualifier;
} else {
- throw new PlanningException("Cannot expect a referenced relation: " + rightExpr);
+ // search for a relation which evaluates a right term included in a join condition
+ for (RelationNode rel : block.getRelations()) {
+ if (rel.getOutSchema().contains(rightExpr)) {
+ String qualifier = rel.getCanonicalName();
+ relationNames[1] = qualifier;
+ }
+ }
+
+ if (relationNames[1] == null) { // if not found
+ throw new PlanningException("Cannot expect a referenced relation: " + rightExpr);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
index 8e91dca..827daee 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java
@@ -135,8 +135,10 @@ public class ProjectionPushDownRule extends
private BiMap<Integer, EvalNode> idToEvalBiMap;
/** Map: Id -> Names */
private LinkedHashMap<Integer, List<String>> idToNamesMap;
- /** Map: Name -> Boolean */
- private LinkedHashMap<String, Boolean> evaluationStateMap;
+ /** Map: Id -> Boolean */
+ private LinkedHashMap<Integer, Boolean> evaluationStateMap;
+ /** Map: alias name -> Id */
+ private LinkedHashMap<String, Integer> aliasMap;
private LogicalPlan plan;
@@ -146,6 +148,7 @@ public class ProjectionPushDownRule extends
idToEvalBiMap = HashBiMap.create();
idToNamesMap = Maps.newLinkedHashMap();
evaluationStateMap = Maps.newLinkedHashMap();
+ aliasMap = Maps.newLinkedHashMap();
}
private int getNextSeqId() {
@@ -153,6 +156,28 @@ public class ProjectionPushDownRule extends
}
/**
+ * If some expression is duplicated, we call an alias indicating the duplicated expression 'native alias'.
+ * This method checks whether a reference is native alias or not.
+ *
+ * @param name The reference name
+ * @return True if the reference is native alias. Otherwise, it will return False.
+ */
+ public boolean isNativeAlias(String name) {
+ return aliasMap.containsKey(name);
+ }
+
+ /**
+ * This method retrieves the name indicating actual expression that an given alias indicate.
+ *
+ * @param name an alias name
+ * @return Real reference name
+ */
+ public String getRealReferenceName(String name) {
+ int refId = aliasMap.get(name);
+ return getPrimaryName(refId);
+ }
+
+ /**
* Add an expression with a specified name, which is usually an alias.
* Later, you can refer this expression by the specified name.
*/
@@ -163,12 +188,19 @@ public class ProjectionPushDownRule extends
if (nameToIdBiMap.containsKey(specifiedName)) {
int refId = nameToIdBiMap.get(specifiedName);
EvalNode found = idToEvalBiMap.get(refId);
- if (found != null && !evalNode.equals(found)) {
- if (found.getType() != EvalType.FIELD && evalNode.getType() != EvalType.FIELD) {
- throw new PlanningException("Duplicate alias: " + evalNode);
- }
- if (found.getType() == EvalType.FIELD) {
- idToEvalBiMap.forcePut(refId, evalNode);
+ if (found != null) {
+ if (evalNode.equals(found)) { // if input expression already exists
+ return specifiedName;
+ } else {
+ // The case where if existing reference name and a given reference name are the same to each other and
+ // existing EvalNode and a given EvalNode is the different
+ if (found.getType() != EvalType.FIELD && evalNode.getType() != EvalType.FIELD) {
+ throw new PlanningException("Duplicate alias: " + evalNode);
+ }
+
+ if (found.getType() == EvalType.FIELD) {
+ idToEvalBiMap.forcePut(refId, evalNode);
+ }
}
}
}
@@ -176,18 +208,19 @@ public class ProjectionPushDownRule extends
int refId;
if (idToEvalBiMap.inverse().containsKey(evalNode)) {
refId = idToEvalBiMap.inverse().get(evalNode);
+ aliasMap.put(specifiedName, refId);
+
} else {
refId = getNextSeqId();
idToEvalBiMap.put(refId, evalNode);
+ TUtil.putToNestedList(idToNamesMap, refId, specifiedName);
+ for (Column column : EvalTreeUtil.findUniqueColumns(evalNode)) {
+ add(new FieldEval(column));
+ }
+ evaluationStateMap.put(refId, false);
}
nameToIdBiMap.put(specifiedName, refId);
- TUtil.putToNestedList(idToNamesMap, refId, specifiedName);
- evaluationStateMap.put(specifiedName, false);
-
- for (Column column : EvalTreeUtil.findUniqueColumns(evalNode)) {
- add(new FieldEval(column));
- }
return specifiedName;
}
@@ -287,7 +320,8 @@ public class ProjectionPushDownRule extends
if (!nameToIdBiMap.containsKey(name)) {
throw new RuntimeException("No Such target name: " + name);
}
- return evaluationStateMap.get(name);
+ int refId = nameToIdBiMap.get(name);
+ return evaluationStateMap.get(refId);
}
public void markAsEvaluated(Target target) {
@@ -296,7 +330,7 @@ public class ProjectionPushDownRule extends
if (!idToNamesMap.containsKey(refId)) {
throw new RuntimeException("No such eval: " + evalNode);
}
- evaluationStateMap.put(target.getCanonicalName(), true);
+ evaluationStateMap.put(refId, true);
}
public Iterator<Target> getFilteredTargets(Set<String> required) {
@@ -305,6 +339,7 @@ public class ProjectionPushDownRule extends
class FilteredTargetIterator implements Iterator<Target> {
List<Target> filtered = TUtil.newList();
+ Iterator<Target> iterator;
public FilteredTargetIterator(Set<String> required) {
for (String name : nameToIdBiMap.keySet()) {
@@ -312,16 +347,17 @@ public class ProjectionPushDownRule extends
filtered.add(getTarget(name));
}
}
+ iterator = filtered.iterator();
}
@Override
public boolean hasNext() {
- return false;
+ return iterator.hasNext();
}
@Override
public Target next() {
- return null;
+ return iterator.next();
}
@Override
@@ -412,8 +448,15 @@ public class ProjectionPushDownRule extends
for (String referenceName : referenceNames) {
Target target = context.targetListMgr.getTarget(referenceName);
- if (context.targetListMgr.isEvaluated(referenceName)) {
- finalTargets.add(new Target(new FieldEval(target.getNamedColumn())));
+ if (target.getEvalTree().getType() == EvalType.CONST) {
+ finalTargets.add(target);
+ } else if (context.targetListMgr.isEvaluated(referenceName)) {
+ if (context.targetListMgr.isNativeAlias(referenceName)) {
+ String realRefName = context.targetListMgr.getRealReferenceName(referenceName);
+ finalTargets.add(new Target(new FieldEval(realRefName, target.getDataType()), referenceName));
+ } else {
+ finalTargets.add(new Target(new FieldEval(target.getNamedColumn())));
+ }
} else if (LogicalPlanner.checkIfBeEvaluatedAtThis(target.getEvalTree(), node)) {
finalTargets.add(target);
context.targetListMgr.markAsEvaluated(target);
@@ -687,12 +730,35 @@ public class ProjectionPushDownRule extends
return node;
}
+ private static void pushDownIfComplexTermInJoinCondition(Context ctx, EvalNode cnf, EvalNode term)
+ throws PlanningException {
+
+ // If one of both terms in a binary operator is a complex expression, the binary operator will require
+ // multiple phases. In this case, join cannot evaluate a binary operator.
+ // So, we should prevent dividing the binary operator into more subexpressions.
+ if (term.getType() != EvalType.FIELD && !(term instanceof BinaryEval)) {
+ String refName = ctx.addExpr(term);
+ EvalTreeUtil.replace(cnf, term, new FieldEval(refName, term.getValueType()));
+ }
+ }
+
public LogicalNode visitJoin(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node,
Stack<LogicalNode> stack) throws PlanningException {
Context newContext = new Context(context);
String joinQualReference = null;
if (node.hasJoinQual()) {
+ for (EvalNode eachQual : AlgebraicUtil.toConjunctiveNormalFormArray(node.getJoinQual())) {
+ if (eachQual instanceof BinaryEval) {
+ BinaryEval binaryQual = (BinaryEval) eachQual;
+
+ for (int i = 0; i < 2; i++) {
+ EvalNode term = binaryQual.getExpr(i);
+ pushDownIfComplexTermInJoinCondition(newContext, eachQual, term);
+ }
+ }
+ }
+
joinQualReference = newContext.addExpr(node.getJoinQual());
newContext.addNecessaryReferences(node.getJoinQual());
}
@@ -874,7 +940,7 @@ public class ProjectionPushDownRule extends
newContext.addExpr(target);
}
- for (Iterator<Target> it = getFilteredTarget(targets, context.requiredSet); it.hasNext();) {
+ for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) {
Target target = it.next();
if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index adc44e3..3d90a79 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -499,7 +499,7 @@ public class TestCreateTable extends QueryTestCaseBase {
// Table created using CTAS
executeString("CREATE TABLE table3 (c1 int, c2 varchar) PARTITION BY COLUMN (c3 int);").close();
- executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2,c3 from table3;").close();
+ executeString("CREATE TABLE table4 AS SELECT c1*c1, c2, c2 as c2_a,c3 from table3;").close();
executeString("CREATE TABLE table2 LIKE table4");
testMsg = "testCreateTableLike1: Table using CTAS test failed";
assertTrue(testMsg, isClonedTable("table4","table2"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
index 100b9ff..08d4503 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
@@ -856,4 +856,52 @@ public class TestJoinQuery extends QueryTestCaseBase {
executeString("DROP TABLE table14 PURGE;");
executeString("DROP TABLE table15 PURGE;");
}
+
+ @Test
+ public void testDifferentTypesJoinCondition() throws Exception {
+ // select * from table20 t3 join table21 t4 on t3.id = t4.id;
+ executeDDL("table1_int8_ddl.sql", "table1", "table20");
+ executeDDL("table1_int4_ddl.sql", "table1", "table21");
+ try {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ } finally {
+ executeString("DROP TABLE table20");
+ executeString("DROP TABLE table21");
+ }
+ }
+
+ @Test
+ public void testComplexJoinCondition1() throws Exception {
+ // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on n1.n_name = upper(n2.n_name);
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public void testComplexJoinCondition2() throws Exception {
+ // select n1.n_nationkey, n1.n_name, upper(n2.n_name) name from nation n1 join nation n2
+ // on n1.n_name = upper(n2.n_name);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public void testComplexJoinCondition3() throws Exception {
+ // select n1.n_nationkey, n1.n_name, n2.n_name from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name);
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public void testComplexJoinCondition4() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql b/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql
index 90be13b..f79b18b 100644
--- a/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql
+++ b/tajo-core/src/test/resources/queries/TestJoinBroadcast/testLeftOuterJoinWithConstantExpr3.sql
@@ -14,4 +14,4 @@ left outer join (
b
on a.c_custkey = b.c_custkey
order by
- c_custkey;
\ No newline at end of file
+ a.c_custkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql
new file mode 100644
index 0000000..0d35cee
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int4_ddl.sql
@@ -0,0 +1,3 @@
+create external table ${0} (id int, name text, score float, type text) using csv
+with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path};
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql
new file mode 100644
index 0000000..3a7a44a
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/table1_int8_ddl.sql
@@ -0,0 +1,3 @@
+create external table ${0} (id bigint, name text, score float, type text) using csv
+with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path};
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql
new file mode 100644
index 0000000..b61ad38
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition1.sql
@@ -0,0 +1,6 @@
+select
+ n1.n_nationkey,
+ n1.n_name,
+ n2.n_name
+from nation n1 join nation n2 on n1.n_name = upper(n2.n_name)
+order by n1.n_nationkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql
new file mode 100644
index 0000000..33effbb
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition2.sql
@@ -0,0 +1,6 @@
+select
+ n1.n_nationkey,
+ n1.n_name,
+ upper(n2.n_name) as name
+from nation n1 join nation n2 on n1.n_name = upper(n2.n_name)
+order by n1.n_nationkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql
new file mode 100644
index 0000000..5674269
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition3.sql
@@ -0,0 +1,6 @@
+select
+ n1.n_nationkey,
+ n1.n_name,
+ n2.n_name
+from nation n1 join nation n2 on lower(n1.n_name) = lower(n2.n_name)
+order by n1.n_nationkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql
new file mode 100644
index 0000000..45d8adf
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testComplexJoinCondition4.sql
@@ -0,0 +1,6 @@
+select
+ n1.n_nationkey,
+ substr(n1.n_name, 1, 4) name1,
+ substr(n2.n_name, 1, 4) name2
+from nation n1 join nation n2 on substr(n1.n_name, 1, 4) = substr(n2.n_name, 1, 4)
+order by n1.n_nationkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql
new file mode 100644
index 0000000..6bd0a4c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testDifferentTypesJoinCondition.sql
@@ -0,0 +1 @@
+select * from table20 t3 join table21 t4 on t3.id = t4.id;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql
index 90be13b..f79b18b 100644
--- a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql
+++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithConstantExpr3.sql
@@ -14,4 +14,4 @@ left outer join (
b
on a.c_custkey = b.c_custkey
order by
- c_custkey;
\ No newline at end of file
+ a.c_custkey;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result
new file mode 100644
index 0000000..e0691a7
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition1.result
@@ -0,0 +1,27 @@
+n_nationkey,n_name,n_name
+-------------------------------
+0,ALGERIA,ALGERIA
+1,ARGENTINA,ARGENTINA
+2,BRAZIL,BRAZIL
+3,CANADA,CANADA
+4,EGYPT,EGYPT
+5,ETHIOPIA,ETHIOPIA
+6,FRANCE,FRANCE
+7,GERMANY,GERMANY
+8,INDIA,INDIA
+9,INDONESIA,INDONESIA
+10,IRAN,IRAN
+11,IRAQ,IRAQ
+12,JAPAN,JAPAN
+13,JORDAN,JORDAN
+14,KENYA,KENYA
+15,MOROCCO,MOROCCO
+16,MOZAMBIQUE,MOZAMBIQUE
+17,PERU,PERU
+18,CHINA,CHINA
+19,ROMANIA,ROMANIA
+20,SAUDI ARABIA,SAUDI ARABIA
+21,VIETNAM,VIETNAM
+22,RUSSIA,RUSSIA
+23,UNITED KINGDOM,UNITED KINGDOM
+24,UNITED STATES,UNITED STATES
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result
new file mode 100644
index 0000000..63289e1
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition2.result
@@ -0,0 +1,27 @@
+n_nationkey,n_name,name
+-------------------------------
+0,ALGERIA,ALGERIA
+1,ARGENTINA,ARGENTINA
+2,BRAZIL,BRAZIL
+3,CANADA,CANADA
+4,EGYPT,EGYPT
+5,ETHIOPIA,ETHIOPIA
+6,FRANCE,FRANCE
+7,GERMANY,GERMANY
+8,INDIA,INDIA
+9,INDONESIA,INDONESIA
+10,IRAN,IRAN
+11,IRAQ,IRAQ
+12,JAPAN,JAPAN
+13,JORDAN,JORDAN
+14,KENYA,KENYA
+15,MOROCCO,MOROCCO
+16,MOZAMBIQUE,MOZAMBIQUE
+17,PERU,PERU
+18,CHINA,CHINA
+19,ROMANIA,ROMANIA
+20,SAUDI ARABIA,SAUDI ARABIA
+21,VIETNAM,VIETNAM
+22,RUSSIA,RUSSIA
+23,UNITED KINGDOM,UNITED KINGDOM
+24,UNITED STATES,UNITED STATES
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result
new file mode 100644
index 0000000..e0691a7
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition3.result
@@ -0,0 +1,27 @@
+n_nationkey,n_name,n_name
+-------------------------------
+0,ALGERIA,ALGERIA
+1,ARGENTINA,ARGENTINA
+2,BRAZIL,BRAZIL
+3,CANADA,CANADA
+4,EGYPT,EGYPT
+5,ETHIOPIA,ETHIOPIA
+6,FRANCE,FRANCE
+7,GERMANY,GERMANY
+8,INDIA,INDIA
+9,INDONESIA,INDONESIA
+10,IRAN,IRAN
+11,IRAQ,IRAQ
+12,JAPAN,JAPAN
+13,JORDAN,JORDAN
+14,KENYA,KENYA
+15,MOROCCO,MOROCCO
+16,MOZAMBIQUE,MOZAMBIQUE
+17,PERU,PERU
+18,CHINA,CHINA
+19,ROMANIA,ROMANIA
+20,SAUDI ARABIA,SAUDI ARABIA
+21,VIETNAM,VIETNAM
+22,RUSSIA,RUSSIA
+23,UNITED KINGDOM,UNITED KINGDOM
+24,UNITED STATES,UNITED STATES
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result
new file mode 100644
index 0000000..325375d
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestJoinQuery/testComplexJoinCondition4.result
@@ -0,0 +1,29 @@
+n_nationkey,name1,name2
+-------------------------------
+0,ALGE,ALGE
+1,ARGE,ARGE
+2,BRAZ,BRAZ
+3,CANA,CANA
+4,EGYP,EGYP
+5,ETHI,ETHI
+6,FRAN,FRAN
+7,GERM,GERM
+8,INDI,INDI
+9,INDO,INDO
+10,IRAN,IRAN
+11,IRAQ,IRAQ
+12,JAPA,JAPA
+13,JORD,JORD
+14,KENY,KENY
+15,MORO,MORO
+16,MOZA,MOZA
+17,PERU,PERU
+18,CHIN,CHIN
+19,ROMA,ROMA
+20,SAUD,SAUD
+21,VIET,VIET
+22,RUSS,RUSS
+23,UNIT,UNIT
+23,UNIT,UNIT
+24,UNIT,UNIT
+24,UNIT,UNIT
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/b3758361/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result b/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result
new file mode 100644
index 0000000..d5b7510
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestJoinQuery/testDifferentTypesJoinCondition.result
@@ -0,0 +1,7 @@
+id,name,score,type,id,name,score,type
+-------------------------------
+1,ooo,1.1,a,1,ooo,1.1,a
+2,ppp,2.3,b,2,ppp,2.3,b
+3,qqq,3.4,c,3,qqq,3.4,c
+4,rrr,4.5,d,4,rrr,4.5,d
+5,xxx,5.6,e,5,xxx,5.6,e
\ No newline at end of file