You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2017/10/05 02:44:26 UTC
calcite git commit: [CALCITE-1995] Remove terms from Filter if
predicates indicate they are always true or false
Repository: calcite
Updated Branches:
refs/heads/master e0eeb1b45 -> a9ac3e486
[CALCITE-1995] Remove terms from Filter if predicates indicate they are always true or false
Also, strengthen range terms like "field >= constant" to point ranges
"field = constant" based on predicates.
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/a9ac3e48
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/a9ac3e48
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/a9ac3e48
Branch: refs/heads/master
Commit: a9ac3e486863b36009a6e5e0f066134ad34b3334
Parents: e0eeb1b
Author: Julian Hyde <jh...@apache.org>
Authored: Sat Sep 30 16:28:29 2017 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Wed Oct 4 10:25:07 2017 -0700
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableCalc.java | 5 +-
.../calcite/plan/SubstitutionVisitor.java | 4 +-
.../calcite/rel/metadata/RelMdPredicates.java | 4 +-
.../rel/rules/AbstractMaterializedViewRule.java | 7 +-
.../rel/rules/FilterProjectTransposeRule.java | 5 +-
.../rel/rules/ReduceExpressionsRule.java | 2 +-
.../java/org/apache/calcite/rex/RexProgram.java | 7 +-
.../apache/calcite/rex/RexProgramBuilder.java | 16 +-
.../org/apache/calcite/rex/RexSimplify.java | 328 ++++++++++++++-----
.../java/org/apache/calcite/rex/RexUtil.java | 42 +--
.../org/apache/calcite/tools/RelBuilder.java | 7 +-
.../calcite/test/MaterializationTest.java | 4 +-
.../apache/calcite/test/RelOptRulesTest.java | 8 +
.../calcite/test/RexImplicationCheckerTest.java | 5 +-
.../org/apache/calcite/test/RexProgramTest.java | 80 ++++-
.../org/apache/calcite/test/RelOptRulesTest.xml | 25 ++
.../calcite/adapter/druid/DruidRules.java | 67 ++--
site/_docs/adapter.md | 8 +-
18 files changed, 475 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java
index 25c882a..9e2018b 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableCalc.java
@@ -28,6 +28,7 @@ import org.apache.calcite.linq4j.tree.MemberDeclaration;
import org.apache.calcite.linq4j.tree.ParameterExpression;
import org.apache.calcite.linq4j.tree.Types;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
@@ -150,8 +151,10 @@ public class EnumerableCalc extends Calc implements EnumerableRel {
inputJavaType);
final RexBuilder rexBuilder = getCluster().getRexBuilder();
+ final RelMetadataQuery mq = RelMetadataQuery.instance();
+ final RelOptPredicateList predicates = mq.getPulledUpPredicates(child);
final RexSimplify simplify =
- new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR);
+ new RexSimplify(rexBuilder, predicates, false, RexUtil.EXECUTOR);
final RexProgram program = this.program.normalize(rexBuilder, simplify);
BlockStatement moveNextBody;
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index 5fd8f6e..ffc036a 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -179,7 +179,9 @@ public class SubstitutionVisitor {
this.cluster = target_.getCluster();
final RexExecutor executor =
Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR);
- this.simplify = new RexSimplify(cluster.getRexBuilder(), false, executor);
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
+ this.simplify =
+ new RexSimplify(cluster.getRexBuilder(), predicates, false, executor);
this.rules = rules;
this.query = Holder.of(MutableRels.toMutable(query_));
this.target = MutableRels.toMutable(target_);
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
index 1b7dbc3..bf3a241 100644
--- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
+++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java
@@ -419,7 +419,9 @@ public class RelMdPredicates
final RelOptCluster cluster = union.getCluster();
final RexExecutor executor =
Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR);
- final RexSimplify simplify = new RexSimplify(rB, true, executor);
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
+ final RexSimplify simplify =
+ new RexSimplify(rB, predicates, true, executor);
RexNode disjPred = simplify.simplifyOrs(finalResidualPreds);
if (!disjPred.isAlwaysTrue()) {
preds.add(disjPred);
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
index 7202b81..d563f8b 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
@@ -41,6 +41,7 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexExecutor;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
@@ -176,9 +177,11 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
final RexBuilder rexBuilder = node.getCluster().getRexBuilder();
final RelMetadataQuery mq = RelMetadataQuery.instance();
final RelOptPlanner planner = call.getPlanner();
+ final RexExecutor executor =
+ Util.first(planner.getExecutor(), RexUtil.EXECUTOR);
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
final RexSimplify simplify =
- new RexSimplify(rexBuilder, true,
- planner.getExecutor() != null ? planner.getExecutor() : RexUtil.EXECUTOR);
+ new RexSimplify(rexBuilder, predicates, true, executor);
final List<RelOptMaterialization> materializations =
(planner instanceof VolcanoPlanner)
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
index d774b2a..aaec8a8 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.rel.rules;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
@@ -119,8 +120,10 @@ public class FilterProjectTransposeRule extends RelOptRule {
final RelBuilder relBuilder = call.builder();
RelNode newFilterRel;
if (copyFilter) {
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
final RexSimplify simplify =
- new RexSimplify(relBuilder.getRexBuilder(), false, RexUtil.EXECUTOR);
+ new RexSimplify(relBuilder.getRexBuilder(), predicates, false,
+ RexUtil.EXECUTOR);
newCondition = simplify.removeNullabilityCast(newCondition);
newFilterRel = filter.copy(filter.getTraitSet(), project.getInput(),
newCondition);
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
index b640c82..b5fd694 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ReduceExpressionsRule.java
@@ -461,7 +461,7 @@ public abstract class ReduceExpressionsRule extends RelOptRule {
final RexExecutor executor =
Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR);
final RexSimplify simplify =
- new RexSimplify(rexBuilder, unknownAsFalse, executor);
+ new RexSimplify(rexBuilder, predicates, unknownAsFalse, executor);
// Simplify predicates in place
boolean reduced = reduceExpressionsInternal(rel, simplify, expList,
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rex/RexProgram.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgram.java b/core/src/main/java/org/apache/calcite/rex/RexProgram.java
index 2adcb0e..c91c0f0 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgram.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgram.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.rex;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
@@ -786,8 +787,10 @@ public class RexProgram {
@Deprecated // to be removed before 2.0
public RexProgram normalize(RexBuilder rexBuilder, boolean simplify) {
- return normalize(rexBuilder,
- simplify ? new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR) : null);
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
+ return normalize(rexBuilder, simplify
+ ? new RexSimplify(rexBuilder, predicates, false, RexUtil.EXECUTOR)
+ : null);
}
//~ Inner Classes ----------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
index aaea0c2..2964a48 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java
@@ -16,6 +16,7 @@
*/
package org.apache.calcite.rex;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -323,7 +324,10 @@ public class RexProgramBuilder {
* sub-expression exists.
*/
private RexLocalRef registerInternal(RexNode expr, boolean force) {
- expr = new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR).simplify(expr);
+ final RexSimplify simplify =
+ new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ RexUtil.EXECUTOR);
+ expr = simplify.simplify(expr);
RexLocalRef ref;
final Pair<String, String> key;
@@ -543,10 +547,14 @@ public class RexProgramBuilder {
final RexNode condition,
final RelDataType outputRowType,
boolean normalize,
- boolean simplify) {
+ boolean simplify_) {
+ RexSimplify simplify = null;
+ if (simplify_) {
+ simplify = new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ RexUtil.EXECUTOR);
+ }
return new RexProgramBuilder(rexBuilder, inputRowType, exprList,
- projectList, condition, outputRowType, normalize,
- simplify ? new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR) : null);
+ projectList, condition, outputRowType, normalize, simplify);
}
@Deprecated // to be removed before 2.0
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
index 286a697..f826c77 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexSimplify.java
@@ -17,6 +17,7 @@
package org.apache.calcite.rex;
import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.Strong;
import org.apache.calcite.rel.core.Project;
@@ -50,6 +51,7 @@ import java.util.Set;
*/
public class RexSimplify {
public final RexBuilder rexBuilder;
+ private final RelOptPredicateList predicates;
final boolean unknownAsFalse;
private final RexExecutor executor;
@@ -57,16 +59,24 @@ public class RexSimplify {
* Creates a RexSimplify.
*
* @param rexBuilder Rex builder
+ * @param predicates Predicates known to hold on input fields
* @param unknownAsFalse Whether to convert UNKNOWN values to FALSE
* @param executor Executor for constant reduction, not null
*/
- public RexSimplify(RexBuilder rexBuilder, boolean unknownAsFalse,
- RexExecutor executor) {
+ public RexSimplify(RexBuilder rexBuilder, RelOptPredicateList predicates,
+ boolean unknownAsFalse, RexExecutor executor) {
this.rexBuilder = Preconditions.checkNotNull(rexBuilder);
+ this.predicates = Preconditions.checkNotNull(predicates);
this.unknownAsFalse = unknownAsFalse;
this.executor = Preconditions.checkNotNull(executor);
}
+ @Deprecated // to be removed before 2.0
+ public RexSimplify(RexBuilder rexBuilder, boolean unknownAsFalse,
+ RexExecutor executor) {
+ this(rexBuilder, RelOptPredicateList.EMPTY, unknownAsFalse, executor);
+ }
+
//~ Methods ----------------------------------------------------------------
/** Returns a RexSimplify the same as this but with a specified
@@ -74,7 +84,15 @@ public class RexSimplify {
public RexSimplify withUnknownAsFalse(boolean unknownAsFalse) {
return unknownAsFalse == this.unknownAsFalse
? this
- : new RexSimplify(rexBuilder, unknownAsFalse, executor);
+ : new RexSimplify(rexBuilder, predicates, unknownAsFalse, executor);
+ }
+
+ /** Returns a RexSimplify the same as this but with a specified
+ * {@link #predicates} value. */
+ public RexSimplify withPredicates(RelOptPredicateList predicates) {
+ return predicates == this.predicates
+ ? this
+ : new RexSimplify(rexBuilder, predicates, unknownAsFalse, executor);
}
/** Simplifies a boolean expression, always preserving its type and its
@@ -146,6 +164,13 @@ public class RexSimplify {
// e must be a comparison (=, >, >=, <, <=, !=)
private RexNode simplifyComparison(RexCall e) {
+ //noinspection unchecked
+ return simplifyComparison(e, Comparable.class);
+ }
+
+ // e must be a comparison (=, >, >=, <, <=, !=)
+ private <C extends Comparable<C>> RexNode simplifyComparison(RexCall e,
+ Class<C> clazz) {
final List<RexNode> operands = new ArrayList<>(e.operands);
simplifyList(operands);
@@ -177,14 +202,13 @@ public class RexSimplify {
if (o0.isA(SqlKind.LITERAL)
&& o1.isA(SqlKind.LITERAL)
&& o0.getType().equals(o1.getType())) {
- final Comparable v0 = ((RexLiteral) o0).getValueAs(Comparable.class);
- final Comparable v1 = ((RexLiteral) o1).getValueAs(Comparable.class);
+ final C v0 = ((RexLiteral) o0).getValueAs(clazz);
+ final C v1 = ((RexLiteral) o1).getValueAs(clazz);
if (v0 == null || v1 == null) {
return unknownAsFalse
? rexBuilder.makeLiteral(false)
: rexBuilder.makeNullLiteral(e.getType());
}
- @SuppressWarnings("unchecked")
final int comparisonResult = v0.compareTo(v1);
switch (e.getKind()) {
case EQUALS:
@@ -205,10 +229,13 @@ public class RexSimplify {
}
// If none of the arguments were simplified, return the call unchanged.
+ final RexNode e2;
if (operands.equals(e.operands)) {
- return e;
+ e2 = e;
+ } else {
+ e2 = rexBuilder.makeCall(e.op, operands);
}
- return rexBuilder.makeCall(e.op, operands);
+ return simplifyUsingPredicates(e2, clazz);
}
/**
@@ -522,10 +549,6 @@ public class RexSimplify {
if (terms.isEmpty() && notTerms.isEmpty()) {
return rexBuilder.makeLiteral(true);
}
- if (terms.size() == 1 && notTerms.isEmpty()) {
- // Make sure "x OR y OR x" (a single-term conjunction) gets simplified.
- return simplify(terms.get(0));
- }
// If one of the not-disjunctions is a disjunction that is wholly
// contained in the disjunctions list, the expression is not
// satisfiable.
@@ -552,6 +575,12 @@ public class RexSimplify {
* returns UNKNOWN it will be interpreted as FALSE. */
RexNode simplifyAnd2ForUnknownAsFalse(List<RexNode> terms,
List<RexNode> notTerms) {
+ //noinspection unchecked
+ return simplifyAnd2ForUnknownAsFalse(terms, notTerms, Comparable.class);
+ }
+
+ private <C extends Comparable<C>> RexNode simplifyAnd2ForUnknownAsFalse(
+ List<RexNode> terms, List<RexNode> notTerms, Class<C> clazz) {
for (RexNode term : terms) {
if (term.isAlwaysFalse()) {
return rexBuilder.makeLiteral(false);
@@ -566,12 +595,31 @@ public class RexSimplify {
}
// Try to simplify the expression
final Multimap<String, Pair<String, RexNode>> equalityTerms = ArrayListMultimap.create();
- final Map<String, Pair<Range, List<RexNode>>> rangeTerms = new HashMap<>();
+ final Map<String, Pair<Range<C>, List<RexNode>>> rangeTerms =
+ new HashMap<>();
final Map<String, String> equalityConstantTerms = new HashMap<>();
final Set<String> negatedTerms = new HashSet<>();
final Set<String> nullOperands = new HashSet<>();
final Set<RexNode> notNullOperands = new LinkedHashSet<>();
final Set<String> comparedOperands = new HashSet<>();
+
+ // Add the predicates from the source to the range terms.
+ for (RexNode predicate : predicates.pulledUpPredicates) {
+ final Comparison comparison = Comparison.of(predicate);
+ if (comparison != null
+ && comparison.kind != SqlKind.NOT_EQUALS) { // not supported yet
+ final C v0 = comparison.literal.getValueAs(clazz);
+ if (v0 != null) {
+ final RexNode result = processRange(rexBuilder, terms, rangeTerms,
+ predicate, comparison.ref, v0, comparison.kind);
+ if (result != null) {
+ // Not satisfiable
+ return result;
+ }
+ }
+ }
+ }
+
for (int i = 0; i < terms.size(); i++) {
RexNode term = terms.get(i);
if (!RexUtil.isDeterministic(term)) {
@@ -613,32 +661,25 @@ public class RexSimplify {
RexCall rightCast = (RexCall) right;
comparedOperands.add(rightCast.getOperands().get(0).toString());
}
- final boolean leftRef = RexUtil.isReferenceOrAccess(left, true);
- final boolean rightRef = RexUtil.isReferenceOrAccess(right, true);
- final boolean leftConstant = left.isA(SqlKind.LITERAL);
- final boolean rightConstant = right.isA(SqlKind.LITERAL);
+ final Comparison comparison = Comparison.of(term);
// Check for comparison with null values
- if (leftConstant && ((RexLiteral) left).getValue() == null
- || rightConstant && ((RexLiteral) right).getValue() == null) {
+ if (comparison != null
+ && comparison.literal.getValue() == null) {
return rexBuilder.makeLiteral(false);
}
// Check for equality on different constants. If the same ref or CAST(ref)
// is equal to different constants, this condition cannot be satisfied,
// and hence it can be evaluated to FALSE
if (term.getKind() == SqlKind.EQUALS) {
- if (leftRef && rightConstant) {
- final String literal = right.toString();
- final String prevLiteral = equalityConstantTerms.put(left.toString(), literal);
+ if (comparison != null) {
+ final String literal = comparison.literal.toString();
+ final String prevLiteral =
+ equalityConstantTerms.put(comparison.ref.toString(), literal);
if (prevLiteral != null && !literal.equals(prevLiteral)) {
return rexBuilder.makeLiteral(false);
}
- } else if (leftConstant && rightRef) {
- final String literal = left.toString();
- final String prevLiteral = equalityConstantTerms.put(right.toString(), literal);
- if (prevLiteral != null && !literal.equals(prevLiteral)) {
- return rexBuilder.makeLiteral(false);
- }
- } else if (leftRef && rightRef) {
+ } else if (RexUtil.isReferenceOrAccess(left, true)
+ && RexUtil.isReferenceOrAccess(right, true)) {
equalityTerms.put(left.toString(), Pair.of(right.toString(), term));
}
}
@@ -655,23 +696,20 @@ public class RexSimplify {
negatedTerms.add(invertNegatedTerm.toString());
}
}
- // Range
- SqlKind comparison = null;
- RexNode ref = null;
- RexLiteral constant = null;
- if (leftRef && rightConstant) {
- comparison = term.getKind();
- ref = left;
- constant = (RexLiteral) right;
- } else if (leftConstant && rightRef) {
- comparison = term.getKind().reverse();
- constant = (RexLiteral) left;
- ref = right;
+ // Remove terms that are implied by predicates on the input,
+ // or weaken terms that are partially implied.
+ // E.g. given predicate "x >= 5" and term "x between 3 and 10"
+ // we weaken to term to "x between 5 and 10".
+ final RexNode term2 = simplifyUsingPredicates(term, clazz);
+ if (term2 != term) {
+ terms.set(i, term = term2);
}
+ // Range
if (comparison != null
- && comparison != SqlKind.NOT_EQUALS) { // NOT_EQUALS not supported
+ && comparison.kind != SqlKind.NOT_EQUALS) { // not supported yet
+ final C constant = comparison.literal.getValueAs(clazz);
final RexNode result = processRange(rexBuilder, terms, rangeTerms,
- term, ref, constant, comparison);
+ term, comparison.ref, constant, comparison.kind);
if (result != null) {
// Not satisfiable
return result;
@@ -739,7 +777,7 @@ public class RexSimplify {
// Example #1. x AND y AND z AND NOT (x AND y) - not satisfiable
// Example #2. x AND y AND NOT (x AND y) - not satisfiable
// Example #3. x AND y AND NOT (x AND y AND z) - may be satisfiable
- final Set<String> termsSet = new HashSet<String>(RexUtil.strings(terms));
+ final Set<String> termsSet = new HashSet<>(RexUtil.strings(terms));
for (RexNode notDisjunction : notTerms) {
if (!RexUtil.isDeterministic(notDisjunction)) {
continue;
@@ -764,6 +802,84 @@ public class RexSimplify {
return RexUtil.composeConjunction(rexBuilder, terms, false);
}
+ private <C extends Comparable<C>> RexNode simplifyUsingPredicates(RexNode e,
+ Class<C> clazz) {
+ final Comparison comparison = Comparison.of(e);
+ // Check for comparison with null values
+ if (comparison == null
+ || comparison.kind == SqlKind.NOT_EQUALS
+ || comparison.literal.getValue() == null) {
+ return e;
+ }
+ final C v0 = comparison.literal.getValueAs(clazz);
+ final Range<C> range = range(comparison.kind, v0);
+ final Range<C> range2 =
+ residue(comparison.ref, range, predicates.pulledUpPredicates,
+ clazz);
+ if (range2 == null) {
+ // Term is impossible to satisfy given these predicates
+ return rexBuilder.makeLiteral(false);
+ } else if (range2.equals(range)) {
+ // no change
+ return e;
+ } else if (range2.equals(Range.all())) {
+ // Term is always satisfied given these predicates
+ return rexBuilder.makeLiteral(true);
+ } else if (range2.lowerEndpoint().equals(range2.upperEndpoint())) {
+ // range is now a point; it's worth simplifying
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, comparison.ref,
+ rexBuilder.makeLiteral(range2.lowerEndpoint(),
+ comparison.literal.getType(), comparison.literal.getTypeName()));
+ } else {
+ // range has been reduced but it's not worth simplifying
+ return e;
+ }
+ }
+
+ /** Weakens a term so that it checks only what is not implied by predicates.
+ *
+ * <p>The term is broken into "ref comparison constant",
+ * for example "$0 < 5".
+ *
+ * <p>Examples:
+ * <ul>
+ *
+ * <li>{@code residue($0 < 10, [$0 < 5])} returns {@code true}
+ *
+ * <li>{@code residue($0 < 10, [$0 < 20, $0 > 0])} returns {@code $0 < 10}
+ * </ul>
+ */
+ private <C extends Comparable<C>> Range<C> residue(RexNode ref, Range<C> r0,
+ List<RexNode> predicates, Class<C> clazz) {
+ for (RexNode predicate : predicates) {
+ switch (predicate.getKind()) {
+ case EQUALS:
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ final RexCall call = (RexCall) predicate;
+ if (call.operands.get(0).equals(ref)
+ && call.operands.get(1) instanceof RexLiteral) {
+ final RexLiteral literal = (RexLiteral) call.operands.get(1);
+ final C c1 = literal.getValueAs(clazz);
+ final Range<C> r1 = range(predicate.getKind(), c1);
+ if (r0.encloses(r1)) {
+ // Given these predicates, term is always satisfied.
+ // e.g. r0 is "$0 < 10", r1 is "$0 < 5"
+ return Range.all();
+ }
+ if (r0.isConnected(r1)) {
+ return r0.intersection(r1);
+ }
+ // Ranges do not intersect. Return null meaning the empty range.
+ return null;
+ }
+ }
+ }
+ return r0;
+ }
+
/** Simplifies OR(x, x) into x, and similar. */
public RexNode simplifyOr(RexCall call) {
assert call.getKind() == SqlKind.OR;
@@ -837,39 +953,20 @@ public class RexSimplify {
return e;
}
- private static RexNode processRange(RexBuilder rexBuilder,
- List<RexNode> terms, Map<String, Pair<Range, List<RexNode>>> rangeTerms,
- RexNode term, RexNode ref, RexLiteral constant, SqlKind comparison) {
- final Comparable v0 = constant.getValueAs(Comparable.class);
- Pair<Range, List<RexNode>> p = rangeTerms.get(ref.toString());
+ private static <C extends Comparable<C>> RexNode processRange(
+ RexBuilder rexBuilder, List<RexNode> terms,
+ Map<String, Pair<Range<C>, List<RexNode>>> rangeTerms, RexNode term,
+ RexNode ref, C v0, SqlKind comparison) {
+ Pair<Range<C>, List<RexNode>> p = rangeTerms.get(ref.toString());
if (p == null) {
- Range r;
- switch (comparison) {
- case EQUALS:
- r = Range.singleton(v0);
- break;
- case LESS_THAN:
- r = Range.lessThan(v0);
- break;
- case LESS_THAN_OR_EQUAL:
- r = Range.atMost(v0);
- break;
- case GREATER_THAN:
- r = Range.greaterThan(v0);
- break;
- case GREATER_THAN_OR_EQUAL:
- r = Range.atLeast(v0);
- break;
- default:
- throw new AssertionError();
- }
rangeTerms.put(ref.toString(),
- new Pair(r, ImmutableList.of(term)));
+ Pair.of(range(comparison, v0),
+ (List<RexNode>) ImmutableList.of(term)));
} else {
// Exists
boolean removeUpperBound = false;
boolean removeLowerBound = false;
- Range r = p.left;
+ Range<C> r = p.left;
switch (comparison) {
case EQUALS:
if (!r.contains(v0)) {
@@ -877,10 +974,11 @@ public class RexSimplify {
return rexBuilder.makeLiteral(false);
}
rangeTerms.put(ref.toString(),
- new Pair(Range.singleton(v0), ImmutableList.of(term)));
+ Pair.of(Range.singleton(v0),
+ (List<RexNode>) ImmutableList.of(term)));
// remove
for (RexNode e : p.right) {
- terms.set(terms.indexOf(e), rexBuilder.makeLiteral(true));
+ Collections.replaceAll(terms, e, rexBuilder.makeLiteral(true));
}
break;
case LESS_THAN: {
@@ -1027,31 +1125,99 @@ public class RexSimplify {
if (removeUpperBound) {
ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder();
for (RexNode e : p.right) {
- if (e.isA(SqlKind.LESS_THAN) || e.isA(SqlKind.LESS_THAN_OR_EQUAL)) {
- terms.set(terms.indexOf(e), rexBuilder.makeLiteral(true));
- } else {
+ switch (e.getKind()) {
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ Collections.replaceAll(terms, e, rexBuilder.makeLiteral(true));
+ break;
+ default:
newBounds.add(e);
}
}
newBounds.add(term);
- rangeTerms.put(ref.toString(), new Pair(r, newBounds.build()));
+ rangeTerms.put(ref.toString(),
+ Pair.of(r, (List<RexNode>) newBounds.build()));
} else if (removeLowerBound) {
ImmutableList.Builder<RexNode> newBounds = ImmutableList.builder();
for (RexNode e : p.right) {
- if (e.isA(SqlKind.GREATER_THAN) || e.isA(SqlKind.GREATER_THAN_OR_EQUAL)) {
- terms.set(terms.indexOf(e), rexBuilder.makeLiteral(true));
- } else {
+ switch (e.getKind()) {
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ Collections.replaceAll(terms, e, rexBuilder.makeLiteral(true));
+ break;
+ default:
newBounds.add(e);
}
}
newBounds.add(term);
- rangeTerms.put(ref.toString(), new Pair(r, newBounds.build()));
+ rangeTerms.put(ref.toString(),
+ Pair.of(r, (List<RexNode>) newBounds.build()));
}
}
// Default
return null;
}
+ private static <C extends Comparable<C>> Range<C> range(SqlKind comparison,
+ C c) {
+ switch (comparison) {
+ case EQUALS:
+ return Range.singleton(c);
+ case LESS_THAN:
+ return Range.lessThan(c);
+ case LESS_THAN_OR_EQUAL:
+ return Range.atMost(c);
+ case GREATER_THAN:
+ return Range.greaterThan(c);
+ case GREATER_THAN_OR_EQUAL:
+ return Range.atLeast(c);
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ /** Comparison between a {@link RexInputRef} or {@link RexFieldAccess} and a
+ * literal. Literal may be on left or right side, and may be null. */
+ private static class Comparison {
+ final RexNode ref;
+ final SqlKind kind;
+ final RexLiteral literal;
+
+ private Comparison(RexNode ref, SqlKind kind, RexLiteral literal) {
+ this.ref = Preconditions.checkNotNull(ref);
+ this.kind = Preconditions.checkNotNull(kind);
+ this.literal = Preconditions.checkNotNull(literal);
+ }
+
+ /** Creates a comparison, or returns null. */
+ static Comparison of(RexNode e) {
+ switch (e.getKind()) {
+ case EQUALS:
+ case NOT_EQUALS:
+ case LESS_THAN:
+ case GREATER_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case GREATER_THAN_OR_EQUAL:
+ final RexCall call = (RexCall) e;
+ final RexNode left = call.getOperands().get(0);
+ final RexNode right = call.getOperands().get(1);
+ switch (right.getKind()) {
+ case LITERAL:
+ if (RexUtil.isReferenceOrAccess(left, true)) {
+ return new Comparison(left, e.getKind(), (RexLiteral) right);
+ }
+ }
+ switch (left.getKind()) {
+ case LITERAL:
+ if (RexUtil.isReferenceOrAccess(right, true)) {
+ return new Comparison(right, e.getKind().reverse(),
+ (RexLiteral) left);
+ }
+ }
+ }
+ return null;
+ }
+ }
}
// End RexSimplify.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/rex/RexUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rex/RexUtil.java b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
index b1e667e..216b27c 100644
--- a/core/src/main/java/org/apache/calcite/rex/RexUtil.java
+++ b/core/src/main/java/org/apache/calcite/rex/RexUtil.java
@@ -17,6 +17,7 @@
package org.apache.calcite.rex;
import org.apache.calcite.linq4j.function.Predicate1;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
@@ -1673,8 +1674,8 @@ public class RexUtil {
@Deprecated // to be removed before 2.0
public static RexNode simplifyPreservingType(RexBuilder rexBuilder,
RexNode e) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplifyPreservingType(e);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplifyPreservingType(e);
}
/**
@@ -1686,8 +1687,8 @@ public class RexUtil {
*/
@Deprecated // to be removed before 2.0
public static RexNode simplify(RexBuilder rexBuilder, RexNode e) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplify(e);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplify(e);
}
/**
@@ -1717,8 +1718,8 @@ public class RexUtil {
@Deprecated // to be removed before 2.0
public static RexNode simplify(RexBuilder rexBuilder, RexNode e,
boolean unknownAsFalse) {
- return new RexSimplify(rexBuilder, unknownAsFalse, EXECUTOR)
- .simplify(e);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY,
+ unknownAsFalse, EXECUTOR).simplify(e);
}
/**
@@ -1730,15 +1731,15 @@ public class RexUtil {
@Deprecated // to be removed before 2.0
public static RexNode simplifyAnds(RexBuilder rexBuilder,
Iterable<? extends RexNode> nodes) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplifyAnds(nodes);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplifyAnds(nodes);
}
@Deprecated // to be removed before 2.0
public static RexNode simplifyAnds(RexBuilder rexBuilder,
Iterable<? extends RexNode> nodes, boolean unknownAsFalse) {
- return new RexSimplify(rexBuilder, unknownAsFalse, EXECUTOR)
- .simplifyAnds(nodes);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY,
+ unknownAsFalse, EXECUTOR).simplifyAnds(nodes);
}
/** Negates a logical expression by adding or removing a NOT. */
@@ -1788,22 +1789,23 @@ public class RexUtil {
@Deprecated // to be removed before 2.0
public static RexNode simplifyAnd(RexBuilder rexBuilder, RexCall e,
boolean unknownAsFalse) {
- return new RexSimplify(rexBuilder, unknownAsFalse, EXECUTOR)
- .simplifyAnd(e);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY,
+ unknownAsFalse, EXECUTOR).simplifyAnd(e);
}
@Deprecated // to be removed before 2.0
public static RexNode simplifyAnd2(RexBuilder rexBuilder,
List<RexNode> terms, List<RexNode> notTerms) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplifyAnd2(terms, notTerms);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplifyAnd2(terms, notTerms);
}
@Deprecated // to be removed before 2.0
public static RexNode simplifyAnd2ForUnknownAsFalse(RexBuilder rexBuilder,
List<RexNode> terms, List<RexNode> notTerms) {
- return new RexSimplify(rexBuilder, true, EXECUTOR)
- .simplifyAnd2ForUnknownAsFalse(terms, notTerms);
+ final Class<Comparable> clazz = Comparable.class;
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, true,
+ EXECUTOR).simplifyAnd2ForUnknownAsFalse(terms, notTerms);
}
public static RexNode negate(RexBuilder rexBuilder, RexCall call) {
@@ -1836,15 +1838,15 @@ public class RexUtil {
@Deprecated // to be removed before 2.0
public static RexNode simplifyOr(RexBuilder rexBuilder, RexCall call) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplifyOr(call);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplifyOr(call);
}
@Deprecated // to be removed before 2.0
public static RexNode simplifyOrs(RexBuilder rexBuilder,
List<RexNode> terms) {
- return new RexSimplify(rexBuilder, false, EXECUTOR)
- .simplifyOrs(terms);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ EXECUTOR).simplifyOrs(terms);
}
/**
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index e55015a..6499f42 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -20,6 +20,7 @@ import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Context;
import org.apache.calcite.plan.Contexts;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptSchema;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelOptUtil;
@@ -184,9 +185,11 @@ public class RelBuilder {
final RexExecutor executor =
Util.first(context.unwrap(RexExecutor.class),
Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR));
- this.simplifier = new RexSimplify(cluster.getRexBuilder(), false, executor);
+ final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
+ this.simplifier =
+ new RexSimplify(cluster.getRexBuilder(), predicates, false, executor);
this.simplifierUnknownAsFalse =
- new RexSimplify(cluster.getRexBuilder(), true, executor);
+ new RexSimplify(cluster.getRexBuilder(), predicates, true, executor);
}
/** Creates a RelBuilder. */
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index 6a3bcc9..20093f1 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -20,6 +20,7 @@ import org.apache.calcite.adapter.java.ReflectiveSchema;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.materialize.MaterializationService;
import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.SubstitutionVisitor;
@@ -118,7 +119,8 @@ public class MaterializationTest {
new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
private final RexBuilder rexBuilder = new RexBuilder(typeFactory);
private final RexSimplify simplify =
- new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR);
+ new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ RexUtil.EXECUTOR);
@Test public void testScan() {
CalciteAssert.that()
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index d9aea0d..9b988ee 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -2618,6 +2618,14 @@ public class RelOptRulesTest extends RelOptTestBase {
transitiveInference(ReduceExpressionsRule.FILTER_INSTANCE);
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-1995">[CALCITE-1995]
+ * Remove predicates from Filter if they can be proved to be always true or
+ * false</a>. */
+ @Test public void testSimplifyFilter() throws Exception {
+ transitiveInference(ReduceExpressionsRule.FILTER_INSTANCE);
+ }
+
@Test public void testPullConstantIntoJoin() throws Exception {
transitiveInference(ReduceExpressionsRule.JOIN_INSTANCE);
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java b/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java
index 4fb7369..074a0f8 100644
--- a/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RexImplicationCheckerTest.java
@@ -19,6 +19,7 @@ package org.apache.calcite.test;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptSchema;
import org.apache.calcite.plan.RexImplicationChecker;
import org.apache.calcite.rel.type.RelDataType;
@@ -431,7 +432,9 @@ public class RexImplicationCheckerTest {
});
executor = holder.get();
- simplify = new RexSimplify(rexBuilder, false, executor);
+ simplify =
+ new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ executor);
checker = new RexImplicationChecker(rexBuilder, executor, rowType);
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RexProgramTest.java b/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
index b0467d6..d42cfbb 100644
--- a/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RexProgramTest.java
@@ -21,6 +21,7 @@ import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.avatica.util.ByteString;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.linq4j.QueryProvider;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.Strong;
import org.apache.calcite.rel.type.RelDataType;
@@ -109,7 +110,8 @@ public class RexProgramTest {
rexBuilder = new RexBuilder(typeFactory);
RexExecutor executor =
new RexExecutorImpl(new DummyTestDataContext());
- simplify = new RexSimplify(rexBuilder, false, executor);
+ simplify =
+ new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false, executor);
trueLiteral = rexBuilder.makeLiteral(true);
falseLiteral = rexBuilder.makeLiteral(false);
final RelDataType intType = typeFactory.createSqlType(SqlTypeName.INTEGER);
@@ -197,6 +199,13 @@ public class RexProgramTest {
equalTo(expected));
}
+ private void checkSimplifyFilter(RexNode node, RelOptPredicateList predicates,
+ String expected) {
+ assertThat(simplify.withUnknownAsFalse(true).withPredicates(predicates)
+ .simplify(node).toString(),
+ equalTo(expected));
+ }
+
/** Returns the number of nodes (including leaves) in a Rex tree. */
private static int nodeCount(RexNode node) {
int n = 1;
@@ -1278,9 +1287,9 @@ public class RexProgramTest {
final RexNode cRef = rexBuilder.makeFieldAccess(range, 2);
final RexNode dRef = rexBuilder.makeFieldAccess(range, 3);
final RexLiteral literal1 = rexBuilder.makeExactLiteral(BigDecimal.ONE);
+ final RexLiteral literal5 = rexBuilder.makeExactLiteral(new BigDecimal(5));
final RexLiteral literal10 = rexBuilder.makeExactLiteral(BigDecimal.TEN);
-
// condition, and the inverse
checkSimplifyFilter(and(le(aRef, literal1), gt(aRef, literal1)),
"false");
@@ -1307,6 +1316,11 @@ public class RexProgramTest {
checkSimplifyFilter(and(gt(aRef, literal10), ge(bRef, literal1), lt(aRef, literal10)),
"false");
+ // one "and" containing three "or"s
+ checkSimplifyFilter(
+ or(gt(aRef, literal10), gt(bRef, literal1), gt(aRef, literal10)),
+ "OR(>(?0.a, 10), >(?0.b, 1))");
+
// case: trailing false and null, remove
checkSimplifyFilter(
case_(aRef, trueLiteral, bRef, trueLiteral, cRef, falseLiteral, dRef, falseLiteral,
@@ -1314,6 +1328,65 @@ public class RexProgramTest {
// condition with null value for range
checkSimplifyFilter(and(gt(aRef, unknownLiteral), ge(bRef, literal1)), "false");
+
+ // range with no predicates;
+ // condition "a > 1 && a < 10 && a < 5" yields "a < 1 && a < 5"
+ checkSimplifyFilter(
+ and(gt(aRef, literal1), lt(aRef, literal10), lt(aRef, literal5)),
+ RelOptPredicateList.EMPTY,
+ "AND(>(?0.a, 1), <(?0.a, 5))");
+
+ // condition "a > 1 && a < 10 && a < 5"
+ // with pre-condition "a > 5"
+ // yields "false"
+ checkSimplifyFilter(
+ and(gt(aRef, literal1), lt(aRef, literal10), lt(aRef, literal5)),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(gt(aRef, literal5))),
+ "false");
+
+ // condition "a > 1 && a < 10 && a <= 5"
+ // with pre-condition "a >= 5"
+ // yields "a = 5"
+ // "a <= 5" would also be correct, just a little less concise.
+ checkSimplifyFilter(
+ and(gt(aRef, literal1), lt(aRef, literal10), le(aRef, literal5)),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(ge(aRef, literal5))),
+ "=(?0.a, 5)");
+
+ // condition "a > 1 && a < 10 && a < 5"
+ // with pre-condition "b < 10 && a > 5"
+ // yields "a > 1 and a < 5"
+ checkSimplifyFilter(
+ and(gt(aRef, literal1), lt(aRef, literal10), lt(aRef, literal5)),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(lt(bRef, literal10), ge(aRef, literal1))),
+ "AND(>(?0.a, 1), <(?0.a, 5))");
+
+ // condition "a > 1"
+ // with pre-condition "b < 10 && a > 5"
+ // yields "true"
+ checkSimplifyFilter(gt(aRef, literal1),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(lt(bRef, literal10), gt(aRef, literal5))),
+ "true");
+
+ // condition "a < 1"
+ // with pre-condition "b < 10 && a > 5"
+ // yields "false"
+ checkSimplifyFilter(lt(aRef, literal1),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(lt(bRef, literal10), gt(aRef, literal5))),
+ "false");
+
+ // condition "a > 5"
+ // with pre-condition "b < 10 && a >= 5"
+ // yields "a > 5"
+ checkSimplifyFilter(gt(aRef, literal5),
+ RelOptPredicateList.of(rexBuilder,
+ ImmutableList.of(lt(bRef, literal10), ge(aRef, literal5))),
+ ">(?0.a, 5)");
}
/** Unit test for
@@ -1794,7 +1867,8 @@ public class RexProgramTest {
}
private RexNode simplify(RexNode e) {
- return new RexSimplify(rexBuilder, false, RexUtil.EXECUTOR).simplify(e);
+ return new RexSimplify(rexBuilder, RelOptPredicateList.EMPTY, false,
+ RexUtil.EXECUTOR).simplify(e);
}
}
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index ecf1e46..596f730 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -4673,6 +4673,31 @@ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$
]]>
</Resource>
</TestCase>
+ <TestCase name="testSimplifyFilter">
+ <Resource name="sql">
+ <![CDATA[select *
+from (select * from sales.emp where deptno > 10)
+where empno > 3 and deptno > 5]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalFilter(condition=[AND(>($0, 3), >($7, 5))])
+ LogicalFilter(condition=[>($7, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8])
+ LogicalFilter(condition=[>($0, 3)])
+ LogicalFilter(condition=[>($7, 10)])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testPullConstantIntoJoin">
<Resource name="sql">
<![CDATA[select *
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
index 562e568..0f4d8b7 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
@@ -18,6 +18,7 @@ package org.apache.calcite.adapter.druid;
import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptUtil;
@@ -197,7 +198,10 @@ public class DruidRules {
final List<RexNode> nonValidPreds = new ArrayList<>();
final RexExecutor executor =
Util.first(cluster.getPlanner().getExecutor(), RexUtil.EXECUTOR);
- final RexSimplify simplify = new RexSimplify(rexBuilder, true, executor);
+ final RelOptPredicateList predicates =
+ call.getMetadataQuery().getPulledUpPredicates(filter.getInput());
+ final RexSimplify simplify =
+ new RexSimplify(rexBuilder, predicates, true, executor);
final RexNode cond = simplify.simplify(filter.getCondition());
if (!canPush(cond)) {
return;
@@ -646,7 +650,7 @@ public class DruidRules {
public void onMatch(RelOptRuleCall call) {
final Aggregate aggregate = call.rel(0);
final Project project = call.rel(1);
- DruidQuery query = call.rel(2);
+ final DruidQuery query = call.rel(2);
if (!DruidQuery.isValidSignature(query.signature() + 'p' + 'a')) {
return;
}
@@ -681,13 +685,15 @@ public class DruidRules {
final RelNode newAggregate = aggregate.copy(aggregate.getTraitSet(),
ImmutableList.of(newProject));
+ final DruidQuery query2;
if (filterRefs.size() > 0) {
- query = optimizeFilteredAggregations(query, (Project) newProject,
- (Aggregate) newAggregate);
+ query2 = optimizeFilteredAggregations(call, query, (Project) newProject,
+ (Aggregate) newAggregate);
} else {
- query = DruidQuery.extendQuery(DruidQuery.extendQuery(query, newProject), newAggregate);
+ final DruidQuery query1 = DruidQuery.extendQuery(query, newProject);
+ query2 = DruidQuery.extendQuery(query1, newAggregate);
}
- call.transformTo(query);
+ call.transformTo(query2);
}
/**
@@ -705,20 +711,33 @@ public class DruidRules {
}
/**
- * Attempts to optimize any aggregations with filters in the DruidQuery by
- * 1. Trying to abstract common filters out into the "filter" field
- * 2. Eliminating expressions that are always true or always false when possible
- * 3. ANDing aggregate filters together with the outer filter to allow for pruning of data
- * Should be called before pushing both the aggregate and project into Druid.
- * Assumes that at least one aggregate call has a filter attached to it
- * */
- private DruidQuery optimizeFilteredAggregations(DruidQuery query, Project project,
- Aggregate aggregate) {
+ * Attempts to optimize any aggregations with filters in the DruidQuery.
+ * Uses the following steps:
+ *
+ * <ol>
+ * <li>Tries to abstract common filters out into the "filter" field;
+ * <li>Eliminates expressions that are always true or always false when
+ * possible;
+ * <li>ANDs aggregate filters together with the outer filter to allow for
+ * pruning of data.
+ * </ol>
+ *
+ * <p>Should be called before pushing both the aggregate and project into
+ * Druid. Assumes that at least one aggregate call has a filter attached to
+ * it. */
+ private DruidQuery optimizeFilteredAggregations(RelOptRuleCall call,
+ DruidQuery query,
+ Project project, Aggregate aggregate) {
Filter filter = null;
- RexBuilder builder = query.getCluster().getRexBuilder();
+ final RexBuilder builder = query.getCluster().getRexBuilder();
final RexExecutor executor =
- Util.first(query.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR);
- RexSimplify simplifier = new RexSimplify(builder, true, executor);
+ Util.first(query.getCluster().getPlanner().getExecutor(),
+ RexUtil.EXECUTOR);
+ final RelNode scan = query.rels.get(0); // first rel is the table scan
+ final RelOptPredicateList predicates =
+ call.getMetadataQuery().getPulledUpPredicates(scan);
+ final RexSimplify simplify =
+ new RexSimplify(builder, predicates, true, executor);
// if the druid query originally contained a filter
boolean containsFilter = false;
@@ -748,14 +767,14 @@ public class DruidRules {
RexNode filterNode = RexUtil.composeDisjunction(builder, disjunctions);
// Erase references to filters
- for (AggregateCall call : aggregate.getAggCallList()) {
- int newFilterArg = call.filterArg;
- if (!call.hasFilter()
+ for (AggregateCall aggCall : aggregate.getAggCallList()) {
+ int newFilterArg = aggCall.filterArg;
+ if (!aggCall.hasFilter()
|| (uniqueFilterRefs.size() == 1 && allHaveFilters) // filters get extracted
|| project.getProjects().get(newFilterArg).isAlwaysTrue()) {
newFilterArg = -1;
}
- newCalls.add(call.copy(call.getArgList(), newFilterArg));
+ newCalls.add(aggCall.copy(aggCall.getArgList(), newFilterArg));
}
aggregate = aggregate.copy(aggregate.getTraitSet(), aggregate.getInput(),
aggregate.indicator, aggregate.getGroupSet(), aggregate.getGroupSets(),
@@ -768,7 +787,7 @@ public class DruidRules {
// Simplify the filter as much as possible
RexNode tempFilterNode = filterNode;
- filterNode = simplifier.simplify(filterNode);
+ filterNode = simplify.simplify(filterNode);
// It's possible that after simplification that the expression is now always false.
// Druid cannot handle such a filter.
@@ -782,7 +801,7 @@ public class DruidRules {
filterNode = tempFilterNode;
}
- filter = LogicalFilter.create(query.rels.get(0), filterNode);
+ filter = LogicalFilter.create(scan, filterNode);
boolean addNewFilter = !filter.getCondition().isAlwaysTrue() && allHaveFilters;
// Assumes that Filter nodes are always right after
http://git-wip-us.apache.org/repos/asf/calcite/blob/a9ac3e48/site/_docs/adapter.md
----------------------------------------------------------------------
diff --git a/site/_docs/adapter.md b/site/_docs/adapter.md
index 3987804..2a4695b 100644
--- a/site/_docs/adapter.md
+++ b/site/_docs/adapter.md
@@ -79,11 +79,11 @@ as implemented by Avatica's
| Property | Description |
|:-------- |:------------|
-| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_DECIMAL">approximateDecimal</a> | Whether approximate results from aggregate functions on `DECIMAL` types are acceptable
-| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_DISTINCT_COUNT">approximateDistinctCount</a> | Whether approximate results from `COUNT(DISTINCT ...)` aggregate functions are acceptable
-| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_TOP_N">approximateTopN</a> | Whether approximate results from "Top N" queries * (`ORDER BY aggFun() DESC LIMIT n`) are acceptable
+| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_DECIMAL">approximateDecimal</a> | Whether approximate results from aggregate functions on `DECIMAL` types are acceptable.
+| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_DISTINCT_COUNT">approximateDistinctCount</a> | Whether approximate results from `COUNT(DISTINCT ...)` aggregate functions are acceptable.
+| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#APPROXIMATE_TOP_N">approximateTopN</a> | Whether approximate results from "Top N" queries (`ORDER BY aggFun() DESC LIMIT n`) are acceptable.
| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#CASE_SENSITIVE">caseSensitive</a> | Whether identifiers are matched case-sensitively. If not specified, value from `lex` is used.
-| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#CONFORMANCE">conformance</a> | SQL conformance level. Values: DEFAULT (the default, similar to PRAGMATIC_2003), ORACLE_10, ORACLE_12, PRAGMATIC_99, PRAGMATIC_2003, STRICT_92, STRICT_99, STRICT_2003, SQL_SERVER_2008.
+| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#CONFORMANCE">conformance</a> | SQL conformance level. Values: DEFAULT (the default, similar to PRAGMATIC_2003), LENIENT, MYSQL_5, ORACLE_10, ORACLE_12, PRAGMATIC_99, PRAGMATIC_2003, STRICT_92, STRICT_99, STRICT_2003, SQL_SERVER_2008.
| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#CREATE_MATERIALIZATIONS">createMaterializations</a> | Whether Calcite should create materializations. Default false.
| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#DEFAULT_NULL_COLLATION">defaultNullCollation</a> | How NULL values should be sorted if neither NULLS FIRST nor NULLS LAST are specified in a query. The default, HIGH, sorts NULL values the same as Oracle.
| <a href="{{ site.apiRoot }}/org/apache/calcite/config/CalciteConnectionProperty.html#DRUID_FETCH">druidFetch</a> | How many rows the Druid adapter should fetch at a time when executing SELECT queries.