You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2014/12/27 09:04:05 UTC
[1/2] incubator-calcite git commit: Detect and merge duplicate
predicates 'AND(x, y, x)' to 'AND(x, y)' in more circumstances
Repository: incubator-calcite
Updated Branches:
refs/heads/master c9e58edf0 -> 011a6b4ce
Detect and merge duplicate predicates 'AND(x, y, x)' to 'AND(x, y)' in more circumstances
Make flattenAnd and flattenOr more efficient on empty lists.
Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/3a225ed4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/3a225ed4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/3a225ed4
Branch: refs/heads/master
Commit: 3a225ed4e13c3d727fe7ae5738d15748665047eb
Parents: c9e58ed
Author: julianhyde <jh...@apache.org>
Authored: Fri Dec 26 22:16:06 2014 -0800
Committer: julianhyde <jh...@apache.org>
Committed: Fri Dec 26 22:56:42 2014 -0800
----------------------------------------------------------------------
.../apache/calcite/rex/RexProgramBuilder.java | 31 +++++++++-
.../java/org/apache/calcite/rex/RexUtil.java | 19 ++++++-
.../apache/calcite/test/RelOptRulesTest.java | 35 ++++++++++++
.../org/apache/calcite/test/RexProgramTest.java | 3 +-
.../org/apache/calcite/tools/PlannerTest.java | 2 +-
.../org/apache/calcite/test/RelOptRulesTest.xml | 59 ++++++++++++++++++++
6 files changed, 142 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index fa4233e..2b70160 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -23,6 +23,9 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -301,9 +304,12 @@ public class RexProgramBuilder {
* sub-expression exists.
*/
private RexLocalRef registerInternal(RexNode expr, boolean force) {
+ expr = simplify(expr);
+
RexLocalRef ref;
- Pair<String, String> key = null;
+ final Pair<String, String> key;
if (expr instanceof RexLocalRef) {
+ key = null;
ref = (RexLocalRef) expr;
} else {
key = RexUtil.makeKey(expr);
@@ -326,7 +332,7 @@ public class RexProgramBuilder {
}
}
- while (true) {
+ for (;;) {
int index = ref.index;
final RexNode expr2 = exprList.get(index);
if (expr2 instanceof RexLocalRef) {
@@ -337,6 +343,27 @@ public class RexProgramBuilder {
}
}
+ /** Simplifies AND(x, x) into x, and similar. */
+ private static RexNode simplify(RexNode node) {
+ switch (node.getKind()) {
+ case AND:
+ case OR:
+ // Convert:
+ // AND(x, x) into x
+ // OR(x, y, x) into OR(x, y)
+ final RexCall call = (RexCall) node;
+ if (!Util.isDistinct(call.getOperands())) {
+ final List<RexNode> list2 =
+ ImmutableList.copyOf(Sets.newLinkedHashSet(call.getOperands()));
+ if (list2.size() == 1) {
+ return list2.get(0);
+ }
+ return new RexCall(call.getType(), call.getOperator(), list2);
+ }
+ }
+ return node;
+ }
+
/**
* Adds an expression to the list of common expressions, and returns a
* reference to the expression. <b>DOES NOT CHECK WHETHER THE EXPRESSION
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/main/java/org/apache/calcite/rex/RexUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
index dd06cb2..8fa85c0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -40,11 +40,14 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Utility methods concerning row-expressions.
@@ -623,9 +626,14 @@ public class RexUtil {
* <p>Treats null nodes as literal TRUE (i.e. ignores them). */
public static ImmutableList<RexNode> flattenAnd(
Iterable<? extends RexNode> nodes) {
+ if (nodes instanceof Collection && ((Collection) nodes).isEmpty()) {
+ // Optimize common case
+ return ImmutableList.of();
+ }
final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ final Set<String> digests = Sets.newHashSet(); // to eliminate duplicates
for (RexNode node : nodes) {
- if (node != null) {
+ if (node != null && digests.add(node.digest)) {
addAnd(builder, node);
}
}
@@ -672,9 +680,16 @@ public class RexUtil {
/** Flattens a list of OR nodes. */
public static ImmutableList<RexNode> flattenOr(
Iterable<? extends RexNode> nodes) {
+ if (nodes instanceof Collection && ((Collection) nodes).isEmpty()) {
+ // Optimize common case
+ return ImmutableList.of();
+ }
final ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ final Set<String> digests = Sets.newHashSet(); // to eliminate duplicates
for (RexNode node : nodes) {
- addOr(builder, node);
+ if (digests.add(node.digest)) {
+ addOr(builder, node);
+ }
}
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index a9c6c49..cd2afe4 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -39,6 +39,7 @@ import org.apache.calcite.rel.rules.CalcMergeRule;
import org.apache.calcite.rel.rules.CoerceInputsRule;
import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
import org.apache.calcite.rel.rules.FilterJoinRule;
+import org.apache.calcite.rel.rules.FilterMergeRule;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.rules.FilterSetOpTransposeRule;
import org.apache.calcite.rel.rules.FilterToCalcRule;
@@ -432,6 +433,40 @@ public class RelOptRulesTest extends RelOptTestBase {
+ " where d.name='Propane'");
}
+ /** Tests that filters are combined if they are identical. */
+ @Test public void testMergeFilter() throws Exception {
+ HepProgram program = new HepProgramBuilder()
+ .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
+ .addRuleInstance(FilterMergeRule.INSTANCE)
+ .build();
+
+ checkPlanning(program,
+ "select name from (\n"
+ + " select *\n"
+ + " from dept\n"
+ + " where deptno = 10)\n"
+ + "where deptno = 10\n");
+ }
+
+ /** Tests that a filters is combined are combined if they are identical,
+ * even if one of them originates in an ON clause of a JOIN. */
+ @Test public void testMergeJoinFilter() throws Exception {
+ HepProgram program = new HepProgramBuilder()
+ .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
+ .addRuleInstance(FilterMergeRule.INSTANCE)
+ .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
+ .build();
+
+ checkPlanning(program,
+ "select * from (\n"
+ + " select d.deptno, e.ename\n"
+ + " from emp as e\n"
+ + " join dept as d\n"
+ + " on e.deptno = d.deptno\n"
+ + " and d.deptno = 10)\n"
+ + "where deptno = 10\n");
+ }
+
@Ignore("cycles")
@Test public void testHeterogeneousConversion() throws Exception {
// This one tests the planner's ability to correctly
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RexProgramTest.java b/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
index aaa6b01..fe31ada 100644
--- a/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
@@ -204,8 +204,7 @@ public class RexProgramTest {
"(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], expr#3=[1], "
+ "expr#4=[+($t0, $t3)], expr#5=[+($t2, $t4)], "
+ "expr#6=[+($t0, $t0)], expr#7=[>($t2, $t0)], "
- + "expr#8=[AND($t7, $t7)], expr#9=[AND($t8, $t7)], "
- + "a=[$t5], b=[$t6], $condition=[$t9])",
+ + "a=[$t5], b=[$t6], $condition=[$t7])",
program);
}
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/test/java/org/apache/calcite/tools/PlannerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/tools/PlannerTest.java b/core/src/test/java/org/apache/calcite/tools/PlannerTest.java
index 6cfb117..9d79e86 100644
--- a/core/src/test/java/org/apache/calcite/tools/PlannerTest.java
+++ b/core/src/test/java/org/apache/calcite/tools/PlannerTest.java
@@ -304,7 +304,7 @@ public class PlannerTest {
"select * from \"emps\" where \"deptno\" < 10\n"
+ "union all\n"
+ "select * from \"emps\" where \"deptno\" < 10 or \"empid\" > 1",
- "[OR(<($1, 10), <($1, 10), >($0, 1))]");
+ "[OR(<($1, 10), >($0, 1))]");
}
/** Unit test that parses, validates, converts and plans. */
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/3a225ed4/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index 23322b5..436e0b2 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -3167,4 +3167,63 @@ LogicalProject(C1=[$0])
]]>
</Resource>
</TestCase>
+ <TestCase name="testMergeFilter">
+ <Resource name="sql">
+ <![CDATA[select name from (
+ select *
+ from dept
+ where deptno = 10)
+where deptno = 10
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(NAME=[$1])
+ LogicalFilter(condition=[=($0, 10)])
+ LogicalProject(DEPTNO=[$0], NAME=[$1])
+ LogicalFilter(condition=[=($0, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(NAME=[$1])
+ LogicalProject(DEPTNO=[$0], NAME=[$1])
+ LogicalFilter(condition=[=($0, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testMergeJoinFilter">
+ <Resource name="sql">
+ <![CDATA[select * from (
+ select d.deptno, e.ename
+ from emp as e
+ join dept as d
+ on e.deptno = d.deptno
+ and d.deptno = 10)
+where deptno = 10
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(DEPTNO=[$0], ENAME=[$1])
+ LogicalFilter(condition=[=($0, 10)])
+ LogicalProject(DEPTNO=[$9], ENAME=[$1])
+ LogicalJoin(condition=[AND(=($7, $9), =($9, 10))], joinType=[inner])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(DEPTNO=[$0], ENAME=[$1])
+ LogicalProject(DEPTNO=[$9], ENAME=[$1])
+ LogicalJoin(condition=[=($7, $9)], joinType=[inner])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalFilter(condition=[=($0, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
</Root>
[2/2] incubator-calcite git commit: [CALCITE-448] FilterIntoJoinRule
creates filters containing invalid RexInputRef
Posted by jh...@apache.org.
[CALCITE-448] FilterIntoJoinRule creates filters containing invalid RexInputRef
Refactor FilterJoinRule so you can supply a predicate rather than sub-classing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/011a6b4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/011a6b4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/011a6b4c
Branch: refs/heads/master
Commit: 011a6b4ce6cc999bab126dfc2fee58c23aaab074
Parents: 3a225ed
Author: julianhyde <jh...@apache.org>
Authored: Fri Dec 26 22:04:28 2014 -0800
Committer: julianhyde <jh...@apache.org>
Committed: Fri Dec 26 23:10:19 2014 -0800
----------------------------------------------------------------------
.../org/apache/calcite/rel/core/Filter.java | 7 +-
.../calcite/rel/rules/FilterJoinRule.java | 105 +++++++++++--------
.../rel/rules/FilterProjectTransposeRule.java | 31 ++++--
.../apache/calcite/rex/RexProgramBuilder.java | 2 +-
.../java/org/apache/calcite/rex/RexUtil.java | 45 +++++++-
.../apache/calcite/test/RelOptRulesTest.java | 43 ++++++++
.../org/apache/calcite/test/RelOptRulesTest.xml | 36 +++++++
7 files changed, 213 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/main/java/org/apache/calcite/rel/core/Filter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Filter.java b/core/src/main/java/org/apache/calcite/rel/core/Filter.java
index ca271db..0dc3009 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Filter.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Filter.java
@@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalcitePrepareImpl;
import org.apache.calcite.rel.RelInput;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelWriter;
@@ -72,7 +73,7 @@ public abstract class Filter extends SingleRel {
assert RexUtil.isFlat(condition) : condition;
this.condition = condition;
// Too expensive for everyday use:
- // assert isValid(true);
+ assert !CalcitePrepareImpl.DEBUG || isValid(true);
}
/**
@@ -110,6 +111,10 @@ public abstract class Filter extends SingleRel {
}
@Override public boolean isValid(boolean fail) {
+ if (RexUtil.isNullabilityCast(getCluster().getTypeFactory(), condition)) {
+ assert !fail : "Cast for just nullability not allowed";
+ return false;
+ }
final RexChecker checker = new RexChecker(getInput().getRowType(), fail);
condition.accept(checker);
if (checker.getFailureCount() > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
index bef2844..bb0b8cc 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
@@ -35,6 +35,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
/**
@@ -42,18 +43,31 @@ import java.util.List;
* within a join node into the join node and/or its children nodes.
*/
public abstract class FilterJoinRule extends RelOptRule {
+ /** Predicate that always returns true. With this predicate, every filter
+ * will be pushed into the ON clause. */
+ public static final Predicate TRUE_PREDICATE =
+ new Predicate() {
+ public boolean apply(Join join, JoinRelType joinType, RexNode exp) {
+ return true;
+ }
+ };
+
+ /** Rule that pushes predicates from a Filter into the Join below them. */
public static final FilterJoinRule FILTER_ON_JOIN =
- new FilterIntoJoinRule(true);
+ new FilterIntoJoinRule(true, RelFactories.DEFAULT_FILTER_FACTORY,
+ RelFactories.DEFAULT_PROJECT_FACTORY, TRUE_PREDICATE);
/** Dumber version of {@link #FILTER_ON_JOIN}. Not intended for production
* use, but keeps some tests working for which {@code FILTER_ON_JOIN} is too
* smart. */
public static final FilterJoinRule DUMB_FILTER_ON_JOIN =
- new FilterIntoJoinRule(false);
+ new FilterIntoJoinRule(false, RelFactories.DEFAULT_FILTER_FACTORY,
+ RelFactories.DEFAULT_PROJECT_FACTORY, TRUE_PREDICATE);
+ /** Rule that pushes predicates in a Join into the inputs to the join. */
public static final FilterJoinRule JOIN =
new JoinConditionPushRule(RelFactories.DEFAULT_FILTER_FACTORY,
- RelFactories.DEFAULT_PROJECT_FACTORY);
+ RelFactories.DEFAULT_PROJECT_FACTORY, TRUE_PREDICATE);
/** Whether to try to strengthen join-type. */
private final boolean smart;
@@ -62,6 +76,11 @@ public abstract class FilterJoinRule extends RelOptRule {
private final RelFactories.ProjectFactory projectFactory;
+ /** Predicate that returns whether a filter is valid in the ON clause of a
+ * join for this particular kind of join. If not, Calcite will push it back to
+ * above the join. */
+ private final Predicate predicate;
+
//~ Constructors -----------------------------------------------------------
/**
@@ -71,10 +90,22 @@ public abstract class FilterJoinRule extends RelOptRule {
protected FilterJoinRule(RelOptRuleOperand operand, String id,
boolean smart, RelFactories.FilterFactory filterFactory,
RelFactories.ProjectFactory projectFactory) {
- super(operand, "PushFilterRule:" + id);
+ this(operand, id, smart, filterFactory, projectFactory, TRUE_PREDICATE);
+ }
+
+ /**
+ * Creates a FilterProjectTransposeRule with an explicit root operand and
+ * factories.
+ */
+ protected FilterJoinRule(RelOptRuleOperand operand, String id,
+ boolean smart, RelFactories.FilterFactory filterFactory,
+ RelFactories.ProjectFactory projectFactory,
+ Predicate predicate) {
+ super(operand, "FilterJoinRule:" + id);
this.smart = smart;
this.filterFactory = filterFactory;
this.projectFactory = projectFactory;
+ this.predicate = predicate;
}
//~ Methods ----------------------------------------------------------------
@@ -218,7 +249,9 @@ public abstract class FilterJoinRule extends RelOptRule {
// create a LogicalFilter on top of the join if needed
RelNode newRel =
- RelOptUtil.createFilter(newJoinRel, aboveFilters, filterFactory);
+ RelOptUtil.createFilter(newJoinRel,
+ RexUtil.fixUp(rexBuilder, aboveFilters, newJoinRel.getRowType()),
+ filterFactory);
call.transformTo(newRel);
}
@@ -236,45 +269,28 @@ public abstract class FilterJoinRule extends RelOptRule {
* @param aboveFilters Filter above Join
* @param joinFilters Filters in join condition
* @param join Join
- *
- * @deprecated Use
- * {@link #validateJoinFilters(java.util.List, java.util.List, org.apache.calcite.rel.core.Join, org.apache.calcite.rel.core.JoinRelType)};
- * very short-term; will be removed before
- * {@link org.apache.calcite.util.Bug#upgrade(String) calcite-0.9.2}.
- */
- protected void validateJoinFilters(List<RexNode> aboveFilters,
- List<RexNode> joinFilters, Join join) {
- validateJoinFilters(aboveFilters, joinFilters, join, join.getJoinType());
- }
-
- /**
- * Validates that target execution framework can satisfy join filters.
- *
- * <p>If the join filter cannot be satisfied (for example, if it is
- * {@code l.c1 > r.c2} and the join only supports equi-join), removes the
- * filter from {@code joinFilters} and adds it to {@code aboveFilters}.
- *
- * <p>The default implementation does nothing; i.e. the join can handle all
- * conditions.
- *
- * @param aboveFilters Filter above Join
- * @param joinFilters Filters in join condition
- * @param join Join
* @param joinType JoinRelType could be different from type in Join due to
* outer join simplification.
*/
protected void validateJoinFilters(List<RexNode> aboveFilters,
List<RexNode> joinFilters, Join join, JoinRelType joinType) {
- return;
+ final Iterator<RexNode> filterIter = joinFilters.iterator();
+ while (filterIter.hasNext()) {
+ RexNode exp = filterIter.next();
+ if (!predicate.apply(join, joinType, exp)) {
+ aboveFilters.add(exp);
+ filterIter.remove();
+ }
+ }
}
/** Rule that pushes parts of the join condition to its inputs. */
public static class JoinConditionPushRule extends FilterJoinRule {
public JoinConditionPushRule(RelFactories.FilterFactory filterFactory,
- RelFactories.ProjectFactory projectFactory) {
+ RelFactories.ProjectFactory projectFactory, Predicate predicate) {
super(RelOptRule.operand(Join.class, RelOptRule.any()),
- "FilterJoinRule:no-filter",
- true, filterFactory, projectFactory);
+ "FilterJoinRule:no-filter", true, filterFactory, projectFactory,
+ predicate);
}
@Override public void onMatch(RelOptRuleCall call) {
@@ -286,19 +302,15 @@ public abstract class FilterJoinRule extends RelOptRule {
/** Rule that tries to push filter expressions into a join
* condition and into the inputs of the join. */
public static class FilterIntoJoinRule extends FilterJoinRule {
- public FilterIntoJoinRule(boolean smart) {
- this(smart, RelFactories.DEFAULT_FILTER_FACTORY,
- RelFactories.DEFAULT_PROJECT_FACTORY);
- }
-
public FilterIntoJoinRule(boolean smart,
RelFactories.FilterFactory filterFactory,
- RelFactories.ProjectFactory projectFactory) {
+ RelFactories.ProjectFactory projectFactory,
+ Predicate predicate) {
super(
- RelOptRule.operand(Filter.class,
- RelOptRule.operand(Join.class, RelOptRule.any())),
- "FilterJoinRule:filter",
- smart, filterFactory, projectFactory);
+ operand(Filter.class,
+ operand(Join.class, RelOptRule.any())),
+ "FilterJoinRule:filter", smart, filterFactory, projectFactory,
+ predicate);
}
@Override public void onMatch(RelOptRuleCall call) {
@@ -307,6 +319,13 @@ public abstract class FilterJoinRule extends RelOptRule {
perform(call, filter, join);
}
}
+
+ /** Predicate that returns whether a filter is valid in the ON clause of a
+ * join for this particular kind of join. If not, Calcite will push it back to
+ * above the join. */
+ public interface Predicate {
+ boolean apply(Join join, JoinRelType joinType, RexNode exp);
+ }
}
// End FilterJoinRule.java
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
index 2f5b720..802cf87 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
@@ -23,8 +23,11 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexUtil;
/**
* Planner rule that pushes
@@ -70,10 +73,10 @@ public class FilterProjectTransposeRule extends RelOptRule {
// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
- final Filter filterRel = call.rel(0);
- final Project projRel = call.rel(1);
+ final Filter filter = call.rel(0);
+ final Project project = call.rel(1);
- if (RexOver.containsOver(projRel.getProjects(), null)) {
+ if (RexOver.containsOver(project.getProjects(), null)) {
// In general a filter cannot be pushed below a windowing calculation.
// Applying the filter before the aggregation function changes
// the results of the windowing invocation.
@@ -85,20 +88,28 @@ public class FilterProjectTransposeRule extends RelOptRule {
// convert the filter to one that references the child of the project
RexNode newCondition =
- RelOptUtil.pushFilterPastProject(filterRel.getCondition(), projRel);
+ RelOptUtil.pushFilterPastProject(filter.getCondition(), project);
+
+ // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
+ // nullable and not-nullable conditions, but a CAST might get in the way of
+ // other rewrites.
+ final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory();
+ if (RexUtil.isNullabilityCast(typeFactory, newCondition)) {
+ newCondition = ((RexCall) newCondition).getOperands().get(0);
+ }
RelNode newFilterRel =
filterFactory == null
- ? filterRel.copy(filterRel.getTraitSet(), projRel.getInput(),
+ ? filter.copy(filter.getTraitSet(), project.getInput(),
newCondition)
- : filterFactory.createFilter(projRel.getInput(), newCondition);
+ : filterFactory.createFilter(project.getInput(), newCondition);
RelNode newProjRel =
projectFactory == null
- ? projRel.copy(projRel.getTraitSet(), newFilterRel,
- projRel.getProjects(), projRel.getRowType())
- : projectFactory.createProject(newFilterRel, projRel.getProjects(),
- projRel.getRowType().getFieldNames());
+ ? project.copy(project.getTraitSet(), newFilterRel,
+ project.getProjects(), project.getRowType())
+ : projectFactory.createProject(newFilterRel, project.getProjects(),
+ project.getRowType().getFieldNames());
call.transformTo(newProjRel);
}
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index 2b70160..f0d5c3d 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -36,7 +36,7 @@ import java.util.Map;
*
* <p>RexProgramBuilder is necessary because a {@link RexProgram} is immutable.
* (The {@link String} class has the same problem: it is immutable, so they
- * introduced {@link StringBuffer}.)
+ * introduced {@link StringBuilder}.)
*/
public class RexProgramBuilder {
//~ Instance fields --------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/main/java/org/apache/calcite/rex/RexUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
index 8fa85c0..c5efd29 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -204,7 +204,21 @@ public class RexUtil {
}
}
- /**
+ /** Returns whether an expression is a cast just for the purposes of
+ * nullability, not changing any other aspect of the type. */
+ public static boolean isNullabilityCast(RelDataTypeFactory typeFactory,
+ RexNode node) {
+ switch (node.getKind()) {
+ case CAST:
+ final RexCall call = (RexCall) node;
+ final RexNode arg0 = call.getOperands().get(0);
+ return SqlTypeUtil.equalSansNullability(typeFactory, arg0.getType(),
+ call.getType());
+ }
+ return false;
+ }
+
+ /**
* Returns whether a given node contains a RexCall with a specified operator
*
* @param operator to look for
@@ -1075,6 +1089,35 @@ public class RexUtil {
return new CnfHelper(rexBuilder).pull(node);
}
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Such differences in nullability occur when expressions are moved
+ * through outer joins.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixUp(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final RelDataType rowType) {
+ final List<RelDataType> typeList = RelOptUtil.getFieldTypeList(rowType);
+ return new RexShuttle() {
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ if (refType2 == rightType) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ // CHECKSTYLE: IGNORE 1
+ }.apply(nodes);
+ }
+
//~ Inner Classes ----------------------------------------------------------
/**
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index cd2afe4..4e50e89 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -24,6 +24,9 @@ import org.apache.calcite.plan.hep.HepProgram;
import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.logical.LogicalTableModify;
import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
@@ -66,6 +69,7 @@ import org.apache.calcite.rel.rules.UnionToDistinctRule;
import org.apache.calcite.rel.rules.ValuesReduceRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidator;
@@ -242,6 +246,45 @@ public class RelOptRulesTest extends RelOptTestBase {
+ "where dept1.c1 > 'c' and (dept1.c2 > 30 or dept1.c1 < 'z')");
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-448">[CALCITE-448],
+ * FilterIntoJoinRule creates filters containing invalid RexInputRef</a>. */
+ @Test public void testPushFilterPastProject() {
+ final HepProgram preProgram =
+ HepProgram.builder()
+ .addRuleInstance(ProjectMergeRule.INSTANCE)
+ .build();
+ final FilterJoinRule.Predicate predicate =
+ new FilterJoinRule.Predicate() {
+ public boolean apply(Join join, JoinRelType joinType, RexNode exp) {
+ return joinType != JoinRelType.INNER;
+ }
+ };
+ final FilterJoinRule join =
+ new FilterJoinRule.JoinConditionPushRule(
+ RelFactories.DEFAULT_FILTER_FACTORY,
+ RelFactories.DEFAULT_PROJECT_FACTORY, predicate);
+ final FilterJoinRule filterOnJoin =
+ new FilterJoinRule.FilterIntoJoinRule(true,
+ RelFactories.DEFAULT_FILTER_FACTORY,
+ RelFactories.DEFAULT_PROJECT_FACTORY, predicate);
+ final HepProgram program =
+ HepProgram.builder()
+ .addGroupBegin()
+ .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
+ .addRuleInstance(join)
+ .addRuleInstance(filterOnJoin)
+ .addGroupEnd()
+ .build();
+ checkPlanning(tester,
+ preProgram,
+ new HepPlanner(program),
+ "select a.name\n"
+ + "from dept a\n"
+ + "left join dept b on b.deptno > 10\n"
+ + "right join dept c on b.deptno > 10\n");
+ }
+
@Test public void testSemiJoinRule() {
final HepProgram preProgram =
HepProgram.builder()
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/011a6b4c/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index 436e0b2..8603a5f 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -3167,6 +3167,42 @@ LogicalProject(C1=[$0])
]]>
</Resource>
</TestCase>
+ <TestCase name="testPushFilterPastProject">
+ <Resource name="sql">
+ <![CDATA[select a.name
+from dept a
+left join dept b on b.deptno > 10
+right join dept c on b.deptno > 10
+]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(NAME=[$1])
+ LogicalJoin(condition=[$4], joinType=[right])
+ LogicalProject(DEPTNO=[$0], NAME=[$1], DEPTNO0=[$2], NAME0=[$3], $f4=[>($2, 10)])
+ LogicalJoin(condition=[$4], joinType=[left])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ LogicalProject(DEPTNO=[$0], NAME=[$1], $f2=[>($0, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(NAME=[$1])
+ LogicalJoin(condition=[true], joinType=[right])
+ LogicalProject(DEPTNO=[$0], NAME=[$1], DEPTNO0=[$2], NAME0=[$3], $f4=[>($2, 10)])
+ LogicalProject(DEPTNO=[$0], NAME=[$1], DEPTNO0=[CAST($2):INTEGER], NAME0=[CAST($3):VARCHAR(10) CHARACTER SET "ISO-8859-1" COLLATE "ISO-8859-1$en_US$primary"], $f2=[CAST($4):BOOLEAN])
+ LogicalJoin(condition=[true], joinType=[inner])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ LogicalProject(DEPTNO=[$0], NAME=[$1], $f2=[>($0, 10)])
+ LogicalFilter(condition=[>($0, 10)])
+ LogicalFilter(condition=[>($0, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testMergeFilter">
<Resource name="sql">
<![CDATA[select name from (