You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2016/10/19 00:36:14 UTC
[57/62] hive git commit: HIVE-13316: Upgrade to Calcite 1.10 (Jesus
Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
index f81c21b..49e4bec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptUtil;
@@ -32,17 +33,28 @@ import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand
public class HiveFilterProjectTSTransposeRule extends RelOptRule {
+ public final static HiveFilterProjectTSTransposeRule INSTANCE =
+ new HiveFilterProjectTSTransposeRule(
+ Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class,
+ HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class);
+
+ public final static HiveFilterProjectTSTransposeRule INSTANCE_DRUID =
+ new HiveFilterProjectTSTransposeRule(
+ Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class,
+ HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class);
+
private final RelFactories.FilterFactory filterFactory;
private final RelFactories.ProjectFactory projectFactory;
- public HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
+ private HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
FilterFactory filterFactory, Class<? extends Project> projectClass,
- ProjectFactory projectFactory, Class<? extends TableScan> tsClass) {
+ ProjectFactory projectFactory, Class<? extends RelNode> tsClass) {
super(operand(filterClass, operand(projectClass, operand(tsClass, none()))));
this.filterFactory = filterFactory;
this.projectFactory = projectFactory;
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
index d43c2c6..91d674d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
@@ -27,8 +27,6 @@ import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories.FilterFactory;
-import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexCall;
@@ -37,6 +35,7 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
@@ -44,25 +43,25 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
public class HiveFilterProjectTransposeRule extends FilterProjectTransposeRule {
public static final HiveFilterProjectTransposeRule INSTANCE_DETERMINISTIC_WINDOWING =
- new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
- HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, true, true);
+ new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
+ HiveRelFactories.HIVE_BUILDER, true, true);
public static final HiveFilterProjectTransposeRule INSTANCE_DETERMINISTIC =
- new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
- HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, true, false);
+ new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
+ HiveRelFactories.HIVE_BUILDER, true, false);
public static final HiveFilterProjectTransposeRule INSTANCE =
- new HiveFilterProjectTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
- HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, false, false);
+ new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
+ HiveRelFactories.HIVE_BUILDER, false, false);
private final boolean onlyDeterministic;
private final boolean pushThroughWindowing;
private HiveFilterProjectTransposeRule(Class<? extends Filter> filterClass,
- FilterFactory filterFactory, Class<? extends Project> projectClass,
- ProjectFactory projectFactory, boolean onlyDeterministic,boolean pushThroughWindowing) {
- super(filterClass, filterFactory, projectClass, projectFactory);
+ Class<? extends Project> projectClass, RelBuilderFactory relBuilderFactory,
+ boolean onlyDeterministic,boolean pushThroughWindowing) {
+ super(filterClass, projectClass, false, false, relBuilderFactory);
this.onlyDeterministic = onlyDeterministic;
this.pushThroughWindowing = pushThroughWindowing;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
index 7c2a7e5..dccb6a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePreFilteringRule.java
@@ -39,7 +39,6 @@ import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +49,7 @@ import com.google.common.collect.Sets;
public class HivePreFilteringRule extends RelOptRule {
- protected static final Logger LOG = LoggerFactory
- .getLogger(HivePreFilteringRule.class
- .getName());
+ protected static final Logger LOG = LoggerFactory.getLogger(HivePreFilteringRule.class);
private static final Set<SqlKind> COMPARISON = EnumSet.of(SqlKind.EQUALS,
SqlKind.GREATER_THAN_OR_EQUAL,
@@ -209,7 +206,7 @@ public class HivePreFilteringRule extends RelOptRule {
for (int i = 0; i < operands.size(); i++) {
final RexNode operand = operands.get(i);
- final RexNode operandCNF = HiveRexUtil.toCnf(rexBuilder, maxCNFNodeCount, operand);
+ final RexNode operandCNF = RexUtil.toCnf(rexBuilder, maxCNFNodeCount, operand);
final List<RexNode> conjunctions = RelOptUtil.conjunctions(operandCNF);
Set<String> refsInCurrentOperand = Sets.newHashSet();
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
index 2fc68ae..a1b5aeb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -16,59 +16,28 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptPredicateList;
-import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.rules.ValuesReduceRule;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rel.rules.ReduceExpressionsRule;
import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexCorrelVariable;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexFieldAccess;
import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexOver;
-import org.apache.calcite.rex.RexRangeRef;
-import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlOperator;
-import org.apache.calcite.sql.fun.SqlRowOperator;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.Pair;
-import org.apache.calcite.util.Stacks;
-import org.apache.calcite.util.Util;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil.ExprSimplifier;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
/**
@@ -82,51 +51,55 @@ import com.google.common.collect.Lists;
* is the same as the type of the resulting cast expression
* </ul>
*/
-public abstract class HiveReduceExpressionsRule extends RelOptRule {
+public abstract class HiveReduceExpressionsRule extends ReduceExpressionsRule {
protected static final Logger LOG = LoggerFactory.getLogger(HiveReduceExpressionsRule.class);
//~ Static fields/initializers ---------------------------------------------
/**
- * Regular expression that matches the description of all instances of this
- * rule and {@link ValuesReduceRule} also. Use
- * it to prevent the planner from invoking these rules.
- */
- public static final Pattern EXCLUSION_PATTERN =
- Pattern.compile("Reduce(Expressions|Values)Rule.*");
-
- /**
* Singleton rule that reduces constants inside a
- * {@link org.apache.calcite.rel.logical.HiveFilter}.
+ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter}.
*/
- public static final HiveReduceExpressionsRule FILTER_INSTANCE =
- new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
+ public static final ReduceExpressionsRule FILTER_INSTANCE =
+ new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
/**
* Singleton rule that reduces constants inside a
- * {@link org.apache.calcite.rel.logical.HiveProject}.
+ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject}.
*/
- public static final HiveReduceExpressionsRule PROJECT_INSTANCE =
+ public static final ReduceExpressionsRule PROJECT_INSTANCE =
new ProjectReduceExpressionsRule(HiveProject.class, HiveRelFactories.HIVE_BUILDER);
/**
* Singleton rule that reduces constants inside a
- * {@link org.apache.calcite.rel.core.HiveJoin}.
+ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin}.
*/
- public static final HiveReduceExpressionsRule JOIN_INSTANCE =
+ public static final ReduceExpressionsRule JOIN_INSTANCE =
new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER);
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveReduceExpressionsRule.
+ *
+ * @param clazz class of rels to which this rule should apply
+ */
+ protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
+ RelBuilderFactory relBuilderFactory, String desc) {
+ super(clazz, relBuilderFactory, desc);
+ }
+
/**
* Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}.
* If the condition is a constant, the filter is removed (if TRUE) or replaced with
* an empty {@link org.apache.calcite.rel.core.Values} (if FALSE or NULL).
*/
- public static class FilterReduceExpressionsRule extends HiveReduceExpressionsRule {
+ public static class FilterReduceExpressionsRule extends ReduceExpressionsRule {
public FilterReduceExpressionsRule(Class<? extends Filter> filterClass,
RelBuilderFactory relBuilderFactory) {
- super(filterClass, relBuilderFactory, "HiveReduceExpressionsRule(Filter)");
+ super(filterClass, relBuilderFactory, "ReduceExpressionsRule(Filter)");
}
@Override public void onMatch(RelOptRuleCall call) {
@@ -135,8 +108,9 @@ public abstract class HiveReduceExpressionsRule extends RelOptRule {
Lists.newArrayList(filter.getCondition());
RexNode newConditionExp;
boolean reduced;
+ final RelMetadataQuery mq = RelMetadataQuery.instance();
final RelOptPredicateList predicates =
- RelMetadataQuery.instance().getPulledUpPredicates(filter.getInput());
+ mq.getPulledUpPredicates(filter.getInput());
if (reduceExpressions(filter, expList, predicates, true)) {
assert expList.size() == 1;
newConditionExp = expList.get(0);
@@ -154,795 +128,95 @@ public abstract class HiveReduceExpressionsRule extends RelOptRule {
// predicate to see if it was already a constant,
// in which case we don't need any runtime decision
// about filtering.
- // TODO: support LogicalValues
if (newConditionExp.isAlwaysTrue()) {
call.transformTo(
filter.getInput());
} else if (reduced) {
+ if (RexUtil.isNullabilityCast(filter.getCluster().getTypeFactory(),
+ newConditionExp)) {
+ newConditionExp = ((RexCall) newConditionExp).getOperands().get(0);
+ }
call.transformTo(call.builder().
push(filter.getInput()).filter(newConditionExp).build());
} else {
+ if (newConditionExp instanceof RexCall) {
+ RexCall rexCall = (RexCall) newConditionExp;
+ boolean reverse = rexCall.getKind() == SqlKind.NOT;
+ if (reverse) {
+ if (!(rexCall.getOperands().get(0) instanceof RexCall)) {
+ // If child is not a RexCall instance, we can bail out
+ return;
+ }
+ rexCall = (RexCall) rexCall.getOperands().get(0);
+ }
+ reduceNotNullableFilter(call, filter, rexCall, reverse);
+ }
return;
}
// New plan is absolutely better than old plan.
call.getPlanner().setImportance(filter, 0.0);
}
- }
-
- /**
- * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}.
- */
- public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule {
-
- public ProjectReduceExpressionsRule(Class<? extends Project> projectClass,
- RelBuilderFactory relBuilderFactory) {
- super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)");
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- Project project = call.rel(0);
- final RelOptPredicateList predicates =
- RelMetadataQuery.instance().getPulledUpPredicates(project.getInput());
- final List<RexNode> expList =
- Lists.newArrayList(project.getProjects());
- if (reduceExpressions(project, expList, predicates)) {
- RelNode newProject = call.builder().push(project.getInput())
- .project(expList, project.getRowType().getFieldNames()).build();
- call.transformTo(newProject);
- // New plan is absolutely better than old plan.
- call.getPlanner().setImportance(project, 0.0);
- }
- }
- }
-
- /**
- * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}.
- */
- public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule {
-
- public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass,
- RelBuilderFactory relBuilderFactory) {
- super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)");
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final HiveJoin join = call.rel(0);
- final List<RexNode> expList = Lists.newArrayList(join.getCondition());
- final int fieldCount = join.getLeft().getRowType().getFieldCount();
- RelMetadataQuery mq = RelMetadataQuery.instance();
- final RelOptPredicateList leftPredicates =
- mq.getPulledUpPredicates(join.getLeft());
- final RelOptPredicateList rightPredicates =
- mq.getPulledUpPredicates(join.getRight());
- final RelOptPredicateList predicates =
- leftPredicates.union(rightPredicates.shift(fieldCount));
- if (!reduceExpressions(join, expList, predicates, true)) {
- return;
- }
- call.transformTo(
- join.copy(
- join.getTraitSet(),
- expList.get(0),
- join.getLeft(),
- join.getRight(),
- join.getJoinType(),
- join.isSemiJoinDone()));
-
- // New plan is absolutely better than old plan.
- call.getPlanner().setImportance(join, 0.0);
- }
- }
-
- //~ Constructors -----------------------------------------------------------
-
- /**
- * Creates a HiveReduceExpressionsRule.
- *
- * @param clazz class of rels to which this rule should apply
- */
- protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
- RelBuilderFactory relBuilderFactory, String desc) {
- super(operand(clazz, any()), relBuilderFactory, desc);
- }
-
- //~ Methods ----------------------------------------------------------------
-
- /**
- * Reduces a list of expressions.
- *
- * @param rel Relational expression
- * @param expList List of expressions, modified in place
- * @param predicates Constraints known to hold on input expressions
- * @return whether reduction found something to change, and succeeded
- */
- protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList,
- RelOptPredicateList predicates) {
- return reduceExpressions(rel, expList, predicates, false);
- }
-
- /**
- * Reduces a list of expressions.
- *
- * @param rel Relational expression
- * @param expList List of expressions, modified in place
- * @param predicates Constraints known to hold on input expressions
- * @param unknownAsFalse Whether UNKNOWN will be treated as FALSE
- *
- * @return whether reduction found something to change, and succeeded
- */
- protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList,
- RelOptPredicateList predicates, boolean unknownAsFalse) {
- RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
-
- boolean reduced = reduceExpressionsInternal(rel, expList, predicates);
-
- // Simplify preds in place
- ExprSimplifier simplifier = new ExprSimplifier(rexBuilder, unknownAsFalse);
- List<RexNode> expList2 = Lists.newArrayList(expList);
- simplifier.mutate(expList2);
- boolean simplified = false;
- for (int i = 0; i < expList.size(); i++) {
- if (!expList2.get(i).toString().equals(expList.get(i).toString())) {
- expList.remove(i);
- expList.add(i, expList2.get(i));
- simplified = true;
- }
- }
-
- return reduced || simplified;
- }
-
- protected static boolean reduceExpressionsInternal(RelNode rel, List<RexNode> expList,
- RelOptPredicateList predicates) {
- RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
-
- // Replace predicates on CASE to CASE on predicates.
- new CaseShuttle().mutate(expList);
-
- // Find reducible expressions.
- final List<RexNode> constExps = Lists.newArrayList();
- List<Boolean> addCasts = Lists.newArrayList();
- final List<RexNode> removableCasts = Lists.newArrayList();
- final ImmutableMap<RexNode, RexNode> constants =
- predicateConstants(RexNode.class, rexBuilder, predicates);
- findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants,
- constExps, addCasts, removableCasts);
- if (constExps.isEmpty() && removableCasts.isEmpty()) {
- return false;
- }
-
- // Remove redundant casts before reducing constant expressions.
- // If the argument to the redundant cast is a reducible constant,
- // reducing that argument to a constant first will result in not being
- // able to locate the original cast expression.
- if (!removableCasts.isEmpty()) {
- final List<RexNode> reducedExprs = Lists.newArrayList();
- for (RexNode exp : removableCasts) {
- RexCall call = (RexCall) exp;
- reducedExprs.add(call.getOperands().get(0));
- }
- RexReplacer replacer =
- new RexReplacer(
- rexBuilder,
- removableCasts,
- reducedExprs,
- Collections.nCopies(removableCasts.size(), false));
- replacer.mutate(expList);
- }
-
- if (constExps.isEmpty()) {
- return true;
- }
-
- final List<RexNode> constExps2 = Lists.newArrayList(constExps);
- if (!constants.isEmpty()) {
- //noinspection unchecked
- final List<Map.Entry<RexNode, RexNode>> pairs =
- (List<Map.Entry<RexNode, RexNode>>) (List)
- Lists.newArrayList(constants.entrySet());
- RexReplacer replacer =
- new RexReplacer(
- rexBuilder,
- Pair.left(pairs),
- Pair.right(pairs),
- Collections.nCopies(pairs.size(), false));
- replacer.mutate(constExps2);
- }
-
- // Compute the values they reduce to.
- RelOptPlanner.Executor executor =
- rel.getCluster().getPlanner().getExecutor();
- if (executor == null) {
- // Cannot reduce expressions: caller has not set an executor in their
- // environment. Caller should execute something like the following before
- // invoking the planner:
- //
- // final RexExecutorImpl executor =
- // new RexExecutorImpl(Schemas.createDataContext(null));
- // rootRel.getCluster().getPlanner().setExecutor(executor);
- return false;
- }
-
- final List<RexNode> reducedValues = Lists.newArrayList();
- executor.reduce(rexBuilder, constExps2, reducedValues);
-
- // Use RexNode.digest to judge whether each newly generated RexNode
- // is equivalent to the original one.
- if (Lists.transform(constExps, HiveCalciteUtil.REX_STR_FN).equals(
- Lists.transform(reducedValues, HiveCalciteUtil.REX_STR_FN))) {
- return false;
- }
-
- // For Project, we have to be sure to preserve the result
- // types, so always cast regardless of the expression type.
- // For other RelNodes like Filter, in general, this isn't necessary,
- // and the presence of casts could hinder other rules such as sarg
- // analysis, which require bare literals. But there are special cases,
- // like when the expression is a UDR argument, that need to be
- // handled as special cases.
- if (rel instanceof Project) {
- addCasts = Collections.nCopies(reducedValues.size(), true);
- }
-
- RexReplacer replacer =
- new RexReplacer(
- rexBuilder,
- constExps,
- reducedValues,
- addCasts);
- replacer.mutate(expList);
- return true;
- }
-
- /**
- * Locates expressions that can be reduced to literals or converted to
- * expressions with redundant casts removed.
- *
- * @param typeFactory Type factory
- * @param exps list of candidate expressions to be examined for
- * reduction
- * @param constants List of expressions known to be constant
- * @param constExps returns the list of expressions that can be constant
- * reduced
- * @param addCasts indicator for each expression that can be constant
- * reduced, whether a cast of the resulting reduced
- * expression is potentially necessary
- * @param removableCasts returns the list of cast expressions where the cast
- */
- protected static void findReducibleExps(RelDataTypeFactory typeFactory,
- List<RexNode> exps, ImmutableMap<RexNode, RexNode> constants,
- List<RexNode> constExps, List<Boolean> addCasts,
- List<RexNode> removableCasts) {
- ReducibleExprLocator gardener =
- new ReducibleExprLocator(typeFactory, constants, constExps,
- addCasts, removableCasts);
- for (RexNode exp : exps) {
- gardener.analyze(exp);
- }
- assert constExps.size() == addCasts.size();
- }
-
- /** Creates a map containing each (e, constant) pair that occurs within
- * a predicate list.
- *
- * @param clazz Class of expression that is considered constant
- * @param rexBuilder Rex builder
- * @param predicates Predicate list
- * @param <C> what to consider a constant: {@link RexLiteral} to use a narrow
- * definition of constant, or {@link RexNode} to use
- * {@link RexUtil#isConstant(RexNode)}
- * @return Map from values to constants
- */
- public static <C extends RexNode> ImmutableMap<RexNode, C> predicateConstants(
- Class<C> clazz, RexBuilder rexBuilder, RelOptPredicateList predicates) {
- // We cannot use an ImmutableMap.Builder here. If there are multiple entries
- // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't
- // matter which we take, so the latter will replace the former.
- // The basic idea is to find all the pairs of RexNode = RexLiteral
- // (1) If 'predicates' contain a non-EQUALS, we bail out.
- // (2) It is OK if a RexNode is equal to the same RexLiteral several times,
- // (e.g. "WHERE deptno = 1 AND deptno = 1")
- // (3) It will return false if there are inconsistent constraints (e.g.
- // "WHERE deptno = 1 AND deptno = 2")
- final Map<RexNode, C> map = new HashMap<>();
- final Set<RexNode> excludeSet = new HashSet<>();
- for (RexNode predicate : predicates.pulledUpPredicates) {
- gatherConstraints(clazz, predicate, map, excludeSet, rexBuilder);
- }
- final ImmutableMap.Builder<RexNode, C> builder =
- ImmutableMap.builder();
- for (Map.Entry<RexNode, C> entry : map.entrySet()) {
- RexNode rexNode = entry.getKey();
- if (!overlap(rexNode, excludeSet)) {
- builder.put(rexNode, entry.getValue());
- }
- }
- return builder.build();
- }
-
- private static <C extends RexNode> void gatherConstraints(Class<C> clazz,
- RexNode predicate, Map<RexNode, C> map, Set<RexNode> excludeSet,
- RexBuilder rexBuilder) {
- if (predicate.getKind() != SqlKind.EQUALS
- && predicate.getKind() != SqlKind.IS_NULL) {
- decompose(excludeSet, predicate);
- return;
- }
- final List<RexNode> operands = ((RexCall) predicate).getOperands();
- if (operands.size() != 2 && predicate.getKind() == SqlKind.EQUALS) {
- decompose(excludeSet, predicate);
- return;
- }
- // if it reaches here, we have rexNode equals rexNode
- final RexNode left;
- final RexNode right;
- if (predicate.getKind() == SqlKind.EQUALS) {
- left = operands.get(0);
- right = operands.get(1);
- } else {
- left = operands.get(0);
- right = rexBuilder.makeNullLiteral(left.getType().getSqlTypeName());
- }
- // note that literals are immutable too and they can only be compared through
- // values.
- gatherConstraint(clazz, left, right, map, excludeSet, rexBuilder);
- gatherConstraint(clazz, right, left, map, excludeSet, rexBuilder);
- }
-
- /** Returns whether a value of {@code type2} can be assigned to a variable
- * of {@code type1}.
- *
- * <p>For example:
- * <ul>
- * <li>{@code canAssignFrom(BIGINT, TINYINT)} returns {@code true}</li>
- * <li>{@code canAssignFrom(TINYINT, BIGINT)} returns {@code false}</li>
- * <li>{@code canAssignFrom(BIGINT, VARCHAR)} returns {@code false}</li>
- * </ul>
- */
- private static boolean canAssignFrom(RelDataType type1, RelDataType type2) {
- final SqlTypeName name1 = type1.getSqlTypeName();
- final SqlTypeName name2 = type2.getSqlTypeName();
- if (name1.getFamily() == name2.getFamily()) {
- switch (name1.getFamily()) {
- case NUMERIC:
- return name1.compareTo(name2) >= 0;
+ /**
+ * For static schema systems, a filter that is always false or null can be
+ * replaced by a values operator that produces no rows, as the schema
+ * information can just be taken from the input Rel. In dynamic schema
+ * environments, the filter might have an unknown input type, in these cases
+ * they must define a system specific alternative to a Values operator, such
+ * as inserting a limit 0 instead of a filter on top of the original input.
+ *
+ * <p>The default implementation of this method is to call
+ * {@link RelBuilder#empty}, which for the static schema will be optimized
+ * to an empty
+ * {@link org.apache.calcite.rel.core.Values}.
+ *
+ * @param input rel to replace, assumes caller has already determined
+ * equivalence to Values operation for 0 records or a
+ * false filter.
+ * @return equivalent but less expensive replacement rel
+ */
+ protected RelNode createEmptyRelOrEquivalent(RelOptRuleCall call, Filter input) {
+ return call.builder().push(input).empty().build();
+ }
+
+ private void reduceNotNullableFilter(
+ RelOptRuleCall call,
+ Filter filter,
+ RexCall rexCall,
+ boolean reverse) {
+ // If the expression is a IS [NOT] NULL on a non-nullable
+ // column, then we can either remove the filter or replace
+ // it with an Empty.
+ boolean alwaysTrue;
+ switch (rexCall.getKind()) {
+ case IS_NULL:
+ case IS_UNKNOWN:
+ alwaysTrue = false;
+ break;
+ case IS_NOT_NULL:
+ alwaysTrue = true;
+ break;
default:
- return true;
- }
- }
- return false;
- }
-
- private static <C extends RexNode> void gatherConstraint(Class<C> clazz,
- RexNode left, RexNode right, Map<RexNode, C> map, Set<RexNode> excludeSet,
- RexBuilder rexBuilder) {
- if (!clazz.isInstance(right)) {
- return;
- }
- if (!RexUtil.isConstant(right)) {
- return;
- }
- C constant = clazz.cast(right);
- if (excludeSet.contains(left)) {
- return;
- }
- final C existedValue = map.get(left);
- if (existedValue == null) {
- switch (left.getKind()) {
- case CAST:
- // Convert "CAST(c) = literal" to "c = literal", as long as it is a
- // widening cast.
- final RexNode operand = ((RexCall) left).getOperands().get(0);
- if (canAssignFrom(left.getType(), operand.getType())) {
- final RexNode castRight =
- rexBuilder.makeCast(operand.getType(), constant);
- if (castRight instanceof RexLiteral) {
- left = operand;
- constant = clazz.cast(castRight);
- }
- }
- }
- map.put(left, constant);
- } else {
- if (existedValue instanceof RexLiteral
- && constant instanceof RexLiteral
- && !((RexLiteral) existedValue).getValue()
- .equals(((RexLiteral) constant).getValue())) {
- // we found conflicting values, e.g. left = 10 and left = 20
- map.remove(left);
- excludeSet.add(left);
- }
- }
- }
-
- private static boolean overlap(RexNode rexNode, Set<RexNode> set) {
- if (rexNode instanceof RexCall) {
- for (RexNode r : ((RexCall) rexNode).getOperands()) {
- if (overlap(r, set)) {
- return true;
- }
- }
- return false;
- } else {
- return set.contains(rexNode);
- }
- }
-
- /** Tries to decompose the RexNode which is a RexCall into non-literal
- * RexNodes. */
- private static void decompose(Set<RexNode> set, RexNode rexNode) {
- if (rexNode instanceof RexCall) {
- for (RexNode r : ((RexCall) rexNode).getOperands()) {
- decompose(set, r);
- }
- } else if (!(rexNode instanceof RexLiteral)) {
- set.add(rexNode);
- }
- }
-
- /** Pushes predicates into a CASE.
- *
- * <p>We have a loose definition of 'predicate': any boolean expression will
- * do, except CASE. For example '(CASE ...) = 5' or '(CASE ...) IS NULL'.
- */
- protected static RexCall pushPredicateIntoCase(RexCall call) {
- if (call.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
- return call;
- }
- switch (call.getKind()) {
- case CASE:
- case AND:
- case OR:
- return call; // don't push CASE into CASE!
- }
- int caseOrdinal = -1;
- final List<RexNode> operands = call.getOperands();
- for (int i = 0; i < operands.size(); i++) {
- RexNode operand = operands.get(i);
- switch (operand.getKind()) {
- case CASE:
- caseOrdinal = i;
- }
- }
- if (caseOrdinal < 0) {
- return call;
- }
- // Convert
- // f(CASE WHEN p1 THEN v1 ... END, arg)
- // to
- // CASE WHEN p1 THEN f(v1, arg) ... END
- final RexCall case_ = (RexCall) operands.get(caseOrdinal);
- final List<RexNode> nodes = new ArrayList<>();
- for (int i = 0; i < case_.getOperands().size(); i++) {
- RexNode node = case_.getOperands().get(i);
- if (!RexUtil.isCasePredicate(case_, i)) {
- node = substitute(call, caseOrdinal, node);
- }
- nodes.add(node);
- }
- return case_.clone(call.getType(), nodes);
- }
-
- /** Converts op(arg0, ..., argOrdinal, ..., argN) to op(arg0,..., node, ..., argN). */
- protected static RexNode substitute(RexCall call, int ordinal, RexNode node) {
- final List<RexNode> newOperands = Lists.newArrayList(call.getOperands());
- newOperands.set(ordinal, node);
- return call.clone(call.getType(), newOperands);
- }
-
- //~ Inner Classes ----------------------------------------------------------
-
- /**
- * Replaces expressions with their reductions. Note that we only have to
- * look for RexCall, since nothing else is reducible in the first place.
- */
- protected static class RexReplacer extends RexShuttle {
- private final RexBuilder rexBuilder;
- private final List<RexNode> reducibleExps;
- private final List<RexNode> reducedValues;
- private final List<Boolean> addCasts;
-
- RexReplacer(
- RexBuilder rexBuilder,
- List<RexNode> reducibleExps,
- List<RexNode> reducedValues,
- List<Boolean> addCasts) {
- this.rexBuilder = rexBuilder;
- this.reducibleExps = reducibleExps;
- this.reducedValues = reducedValues;
- this.addCasts = addCasts;
- }
-
- @Override public RexNode visitInputRef(RexInputRef inputRef) {
- RexNode node = visit(inputRef);
- if (node == null) {
- return super.visitInputRef(inputRef);
- }
- return node;
- }
-
- @Override public RexNode visitCall(RexCall call) {
- RexNode node = visit(call);
- if (node != null) {
- return node;
- }
- node = super.visitCall(call);
- if (node != call) {
- node = HiveRexUtil.simplify(rexBuilder, node);
- }
- return node;
- }
-
- private RexNode visit(final RexNode call) {
- int i = reducibleExps.indexOf(call);
- if (i == -1) {
- return null;
- }
- RexNode replacement = reducedValues.get(i);
- if (addCasts.get(i)
- && (replacement.getType() != call.getType())) {
- // Handle change from nullable to NOT NULL by claiming
- // that the result is still nullable, even though
- // we know it isn't.
- //
- // Also, we cannot reduce CAST('abc' AS VARCHAR(4)) to 'abc'.
- // If we make 'abc' of type VARCHAR(4), we may later encounter
- // the same expression in a Project's digest where it has
- // type VARCHAR(3), and that's wrong.
- replacement = rexBuilder.makeAbstractCast(call.getType(), replacement);
- }
- return replacement;
- }
- }
-
- /**
- * Helper class used to locate expressions that either can be reduced to
- * literals or contain redundant casts.
- */
- protected static class ReducibleExprLocator extends RexVisitorImpl<Void> {
- /** Whether an expression is constant, and if so, whether it can be
- * reduced to a simpler constant. */
- enum Constancy {
- NON_CONSTANT, REDUCIBLE_CONSTANT, IRREDUCIBLE_CONSTANT
- }
-
- private final RelDataTypeFactory typeFactory;
-
- private final List<Constancy> stack;
-
- private final ImmutableMap<RexNode, RexNode> constants;
-
- private final List<RexNode> constExprs;
-
- private final List<Boolean> addCasts;
-
- private final List<RexNode> removableCasts;
-
- private final List<SqlOperator> parentCallTypeStack;
-
- ReducibleExprLocator(RelDataTypeFactory typeFactory,
- ImmutableMap<RexNode, RexNode> constants, List<RexNode> constExprs,
- List<Boolean> addCasts, List<RexNode> removableCasts) {
- // go deep
- super(true);
- this.typeFactory = typeFactory;
- this.constants = constants;
- this.constExprs = constExprs;
- this.addCasts = addCasts;
- this.removableCasts = removableCasts;
- this.stack = Lists.newArrayList();
- this.parentCallTypeStack = Lists.newArrayList();
- }
-
- public void analyze(RexNode exp) {
- assert stack.isEmpty();
-
- exp.accept(this);
-
- // Deal with top of stack
- assert stack.size() == 1;
- assert parentCallTypeStack.isEmpty();
- Constancy rootConstancy = stack.get(0);
- if (rootConstancy == Constancy.REDUCIBLE_CONSTANT) {
- // The entire subtree was constant, so add it to the result.
- addResult(exp);
- }
- stack.clear();
- }
-
- private Void pushVariable() {
- stack.add(Constancy.NON_CONSTANT);
- return null;
- }
-
- private void addResult(RexNode exp) {
- // Cast of literal can't be reduced, so skip those (otherwise we'd
- // go into an infinite loop as we add them back).
- if (exp.getKind() == SqlKind.CAST) {
- RexCall cast = (RexCall) exp;
- RexNode operand = cast.getOperands().get(0);
- if (operand instanceof RexLiteral) {
- return;
- }
- }
- constExprs.add(exp);
-
- // In the case where the expression corresponds to a UDR argument,
- // we need to preserve casts. Note that this only applies to
- // the topmost argument, not expressions nested within the UDR
- // call.
- //
- // REVIEW zfong 6/13/08 - Are there other expressions where we
- // also need to preserve casts?
- if (parentCallTypeStack.isEmpty()) {
- addCasts.add(false);
- } else {
- addCasts.add(isUdf(Stacks.peek(parentCallTypeStack)));
- }
- }
-
- private Boolean isUdf(SqlOperator operator) {
- // return operator instanceof UserDefinedRoutine
- return false;
- }
-
- public Void visitInputRef(RexInputRef inputRef) {
- if (constants.containsKey(inputRef)) {
- stack.add(Constancy.REDUCIBLE_CONSTANT);
- return null;
- }
- return pushVariable();
- }
-
- public Void visitLiteral(RexLiteral literal) {
- stack.add(Constancy.IRREDUCIBLE_CONSTANT);
- return null;
- }
-
- public Void visitOver(RexOver over) {
- // assume non-constant (running SUM(1) looks constant but isn't)
- analyzeCall(over, Constancy.NON_CONSTANT);
- return null;
- }
-
- public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
- return pushVariable();
- }
-
- public Void visitCall(RexCall call) {
- // assume REDUCIBLE_CONSTANT until proven otherwise
- analyzeCall(call, Constancy.REDUCIBLE_CONSTANT);
- return null;
- }
-
- private void analyzeCall(RexCall call, Constancy callConstancy) {
- Stacks.push(parentCallTypeStack, call.getOperator());
-
- // visit operands, pushing their states onto stack
- super.visitCall(call);
-
- // look for NON_CONSTANT operands
- int operandCount = call.getOperands().size();
- List<Constancy> operandStack = Util.last(stack, operandCount);
- for (Constancy operandConstancy : operandStack) {
- if (operandConstancy == Constancy.NON_CONSTANT) {
- callConstancy = Constancy.NON_CONSTANT;
- }
- }
-
- // Even if all operands are constant, the call itself may
- // be non-deterministic.
- if (!call.getOperator().isDeterministic()) {
- callConstancy = Constancy.NON_CONSTANT;
- } else if (call.getOperator().isDynamicFunction()) {
- // We can reduce the call to a constant, but we can't
- // cache the plan if the function is dynamic.
- // For now, treat it same as non-deterministic.
- callConstancy = Constancy.NON_CONSTANT;
- }
-
- // Row operator itself can't be reduced to a literal, but if
- // the operands are constants, we still want to reduce those
- if ((callConstancy == Constancy.REDUCIBLE_CONSTANT)
- && (call.getOperator() instanceof SqlRowOperator)) {
- callConstancy = Constancy.NON_CONSTANT;
+ return;
}
-
- if (callConstancy == Constancy.NON_CONSTANT) {
- // any REDUCIBLE_CONSTANT children are now known to be maximal
- // reducible subtrees, so they can be added to the result
- // list
- for (int iOperand = 0; iOperand < operandCount; ++iOperand) {
- Constancy constancy = operandStack.get(iOperand);
- if (constancy == Constancy.REDUCIBLE_CONSTANT) {
- addResult(call.getOperands().get(iOperand));
+ if (reverse) {
+ alwaysTrue = !alwaysTrue;
+ }
+ RexNode operand = rexCall.getOperands().get(0);
+ if (operand instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) operand;
+ if (!inputRef.getType().isNullable()) {
+ if (alwaysTrue) {
+ call.transformTo(filter.getInput());
+ } else {
+ call.transformTo(createEmptyRelOrEquivalent(call, filter));
}
}
-
- // if this cast expression can't be reduced to a literal,
- // then see if we can remove the cast
- if (call.getOperator() == SqlStdOperatorTable.CAST) {
- reduceCasts(call);
- }
}
-
- // pop operands off of the stack
- operandStack.clear();
-
- // pop this parent call operator off the stack
- Stacks.pop(parentCallTypeStack, call.getOperator());
-
- // push constancy result for this call onto stack
- stack.add(callConstancy);
- }
-
- private void reduceCasts(RexCall outerCast) {
- List<RexNode> operands = outerCast.getOperands();
- if (operands.size() != 1) {
- return;
- }
- RelDataType outerCastType = outerCast.getType();
- RelDataType operandType = operands.get(0).getType();
- if (operandType.equals(outerCastType)) {
- removableCasts.add(outerCast);
- return;
- }
-
- // See if the reduction
- // CAST((CAST x AS type) AS type NOT NULL)
- // -> CAST(x AS type NOT NULL)
- // applies. TODO jvs 15-Dec-2008: consider
- // similar cases for precision changes.
- if (!(operands.get(0) instanceof RexCall)) {
- return;
- }
- RexCall innerCast = (RexCall) operands.get(0);
- if (innerCast.getOperator() != SqlStdOperatorTable.CAST) {
- return;
- }
- if (innerCast.getOperands().size() != 1) {
- return;
- }
- RelDataType outerTypeNullable =
- typeFactory.createTypeWithNullability(outerCastType, true);
- RelDataType innerTypeNullable =
- typeFactory.createTypeWithNullability(operandType, true);
- if (outerTypeNullable != innerTypeNullable) {
- return;
- }
- if (operandType.isNullable()) {
- removableCasts.add(innerCast);
- }
- }
-
- public Void visitDynamicParam(RexDynamicParam dynamicParam) {
- return pushVariable();
- }
-
- public Void visitRangeRef(RexRangeRef rangeRef) {
- return pushVariable();
- }
-
- public Void visitFieldAccess(RexFieldAccess fieldAccess) {
- return pushVariable();
}
}
- /** Shuttle that pushes predicates into a CASE. */
- protected static class CaseShuttle extends RexShuttle {
- @Override public RexNode visitCall(RexCall call) {
- for (;;) {
- call = (RexCall) super.visitCall(call);
- final RexCall old = call;
- call = pushPredicateIntoCase(call);
- if (call == old) {
- return call;
- }
- }
- }
- }
}
// End HiveReduceExpressionsRule.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java
index ec488fe..0644f0c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsWithStatsRule.java
@@ -38,7 +38,6 @@ import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.util.Pair;
import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
@@ -131,7 +130,7 @@ public class HiveReduceExpressionsWithStatsRule extends RelOptRule {
&& call.operands.get(0) instanceof RexLiteral) {
ref = (RexInputRef) call.operands.get(1);
literal = (RexLiteral) call.operands.get(0);
- kind = HiveRexUtil.invert(call.getOperator().getKind());
+ kind = call.getOperator().getKind().reverse();
}
// Found an expression that we can try to reduce
@@ -252,7 +251,7 @@ public class HiveReduceExpressionsWithStatsRule extends RelOptRule {
// If we did not reduce, check the children nodes
RexNode node = super.visitCall(call);
if (node != call) {
- node = HiveRexUtil.simplify(rexBuilder, node);
+ node = RexUtil.simplify(rexBuilder, node);
}
return node;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
index b0cb8df..458fee7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelFieldTrimmer.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
@@ -24,28 +25,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.RelCollation;
-import org.apache.calcite.rel.RelCollations;
-import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexCorrelVariable;
import org.apache.calcite.rex.RexFieldAccess;
-import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPermuteInputsShuttle;
-import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitor;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql2rel.CorrelationReferenceFinder;
@@ -62,7 +57,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo;
import com.google.common.collect.ImmutableList;
@@ -72,11 +66,9 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer {
protected static final Log LOG = LogFactory.getLog(HiveRelFieldTrimmer.class);
- private RelBuilder relBuilder;
-
private ColumnAccessInfo columnAccessInfo;
-
private Map<HiveProject, Table> viewProjectToTableSchema;
+ private final RelBuilder relBuilder;
public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) {
super(validator, relBuilder);
@@ -86,9 +78,9 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer {
public HiveRelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder,
ColumnAccessInfo columnAccessInfo, Map<HiveProject, Table> viewToTableSchema) {
super(validator, relBuilder);
- this.relBuilder = relBuilder;
this.columnAccessInfo = columnAccessInfo;
this.viewProjectToTableSchema = viewToTableSchema;
+ this.relBuilder = relBuilder;
}
/**
@@ -193,186 +185,73 @@ public class HiveRelFieldTrimmer extends RelFieldTrimmer {
/**
* Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
- * {@link org.apache.calcite.rel.core.Sort}.
+ * {@link org.apache.calcite.adapter.druid.DruidQuery}.
*/
- public TrimResult trimFields(
- HiveSortLimit sort,
- ImmutableBitSet fieldsUsed,
+ public TrimResult trimFields(DruidQuery dq, ImmutableBitSet fieldsUsed,
Set<RelDataTypeField> extraFields) {
- final RelDataType rowType = sort.getRowType();
- final int fieldCount = rowType.getFieldCount();
- final RelCollation collation = sort.getCollation();
- final RelNode input = sort.getInput();
- RelOptCluster cluster = sort.getCluster();
-
- // We use the fields used by the consumer, plus any fields used as sort
- // keys.
- final ImmutableBitSet.Builder inputFieldsUsed =
- ImmutableBitSet.builder(fieldsUsed);
- for (RelFieldCollation field : collation.getFieldCollations()) {
- inputFieldsUsed.set(field.getFieldIndex());
+ final int fieldCount = dq.getRowType().getFieldCount();
+ if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount))
+ && extraFields.isEmpty()) {
+ // if there is nothing to project or if we are projecting everything
+ // then no need to introduce another RelNode
+ return trimFields(
+ (RelNode) dq, fieldsUsed, extraFields);
}
-
- // Create input with trimmed columns.
- final Set<RelDataTypeField> inputExtraFields = Collections.emptySet();
- TrimResult trimResult =
- trimChild(sort, input, inputFieldsUsed.build(), inputExtraFields);
- RelNode newInput = trimResult.left;
- final Mapping inputMapping = trimResult.right;
-
- // If the input is unchanged, and we need to project all columns,
- // there's nothing we can do.
- if (newInput == input
- && inputMapping.isIdentity()
- && fieldsUsed.cardinality() == fieldCount) {
- return result(sort, Mappings.createIdentity(fieldCount));
+ final RelNode newTableAccessRel = project(dq, fieldsUsed, extraFields, relBuilder);
+
+ // Some parts of the system can't handle rows with zero fields, so
+ // pretend that one field is used.
+ if (fieldsUsed.cardinality() == 0) {
+ RelNode input = newTableAccessRel;
+ if (input instanceof Project) {
+ // The table has implemented the project in the obvious way - by
+ // creating project with 0 fields. Strip it away, and create our own
+ // project with one field.
+ Project project = (Project) input;
+ if (project.getRowType().getFieldCount() == 0) {
+ input = project.getInput();
+ }
+ }
+ return dummyProject(fieldCount, input);
}
- relBuilder.push(newInput);
- final int offset =
- sort.offset == null ? 0 : RexLiteral.intValue(sort.offset);
- final int fetch =
- sort.fetch == null ? -1 : RexLiteral.intValue(sort.fetch);
- final ImmutableList<RexNode> fields =
- relBuilder.fields(RexUtil.apply(inputMapping, collation));
-
- // The result has the same mapping as the input gave us. Sometimes we
- // return fields that the consumer didn't ask for, because the filter
- // needs them for its condition.
- // TODO: Calcite will return empty LogicalValues when offset == 0 && fetch == 0.
- // However, Hive ASTConverter can not deal with LogicalValues.
- sortLimit(cluster, relBuilder, offset, fetch, fields);
- return result(relBuilder.build(), inputMapping);
+ final Mapping mapping = createMapping(fieldsUsed, fieldCount);
+ return result(newTableAccessRel, mapping);
}
-
- private List<RexNode> projects(RelDataType inputRowType, RelOptCluster cluster) {
- final List<RexNode> exprList = new ArrayList<>();
- for (RelDataTypeField field : inputRowType.getFieldList()) {
- final RexBuilder rexBuilder = cluster.getRexBuilder();
- exprList.add(rexBuilder.makeInputRef(field.getType(), field.getIndex()));
+
+ private static RelNode project(DruidQuery dq, ImmutableBitSet fieldsUsed,
+ Set<RelDataTypeField> extraFields, RelBuilder relBuilder) {
+ final int fieldCount = dq.getRowType().getFieldCount();
+ if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount))
+ && extraFields.isEmpty()) {
+ return dq;
}
- return exprList;
- }
-
- private static RelFieldCollation collation(RexNode node,
- RelFieldCollation.Direction direction,
- RelFieldCollation.NullDirection nullDirection, List<RexNode> extraNodes) {
- switch (node.getKind()) {
- case INPUT_REF:
- return new RelFieldCollation(((RexInputRef) node).getIndex(), direction,
- Util.first(nullDirection, direction.defaultNullDirection()));
- case DESCENDING:
- return collation(((RexCall) node).getOperands().get(0),
- RelFieldCollation.Direction.DESCENDING,
- nullDirection, extraNodes);
- case NULLS_FIRST:
- return collation(((RexCall) node).getOperands().get(0), direction,
- RelFieldCollation.NullDirection.FIRST, extraNodes);
- case NULLS_LAST:
- return collation(((RexCall) node).getOperands().get(0), direction,
- RelFieldCollation.NullDirection.LAST, extraNodes);
- default:
- final int fieldIndex = extraNodes.size();
- extraNodes.add(node);
- return new RelFieldCollation(fieldIndex, direction,
- Util.first(nullDirection, direction.defaultNullDirection()));
+ final List<RexNode> exprList = new ArrayList<>();
+ final List<String> nameList = new ArrayList<>();
+ final RexBuilder rexBuilder = dq.getCluster().getRexBuilder();
+ final List<RelDataTypeField> fields = dq.getRowType().getFieldList();
+
+ // Project the subset of fields.
+ for (int i : fieldsUsed) {
+ RelDataTypeField field = fields.get(i);
+ exprList.add(rexBuilder.makeInputRef(dq, i));
+ nameList.add(field.getName());
}
- }
-
- private void sortLimit(RelOptCluster cluster, RelBuilder relBuilder, int offset, int fetch,
- Iterable<? extends RexNode> nodes) {
- final List<RelFieldCollation> fieldCollations = new ArrayList<>();
- final RelDataType inputRowType = relBuilder.peek().getRowType();
- final List<RexNode> extraNodes = projects(inputRowType, cluster);
- final List<RexNode> originalExtraNodes = ImmutableList.copyOf(extraNodes);
- for (RexNode node : nodes) {
- fieldCollations.add(
- collation(node, RelFieldCollation.Direction.ASCENDING,
- RelFieldCollation.NullDirection.FIRST, extraNodes));
- }
- final RexNode offsetNode = offset <= 0 ? null : relBuilder.literal(offset);
- final RexNode fetchNode = fetch < 0 ? null : relBuilder.literal(fetch);
- if (offsetNode == null && fetchNode == null && fieldCollations.isEmpty()) {
- return; // sort is trivial
- }
- final boolean addedFields = extraNodes.size() > originalExtraNodes.size();
- if (fieldCollations.isEmpty()) {
- assert !addedFields;
- RelNode top = relBuilder.peek();
- if (top instanceof Sort) {
- final Sort sort2 = (Sort) top;
- if (sort2.offset == null && sort2.fetch == null) {
- relBuilder.build();
- relBuilder.push(sort2.getInput());
- final RelNode sort =
- HiveSortLimit.create(relBuilder.build(), sort2.collation,
- offsetNode, fetchNode);
- relBuilder.push(sort);
- return;
- }
- }
- if (top instanceof Project) {
- final Project project = (Project) top;
- if (project.getInput() instanceof Sort) {
- final Sort sort2 = (Sort) project.getInput();
- if (sort2.offset == null && sort2.fetch == null) {
- relBuilder.build();
- relBuilder.push(sort2.getInput());
- final RelNode sort =
- HiveSortLimit.create(relBuilder.build(), sort2.collation,
- offsetNode, fetchNode);
- relBuilder.push(sort);
- relBuilder.project(project.getProjects());
- return;
- }
- }
- }
- }
- if (addedFields) {
- relBuilder.project(extraNodes);
- }
- final RelNode sort =
- HiveSortLimit.create(relBuilder.build(), RelCollations.of(fieldCollations),
- offsetNode, fetchNode);
- relBuilder.push(sort);
- if (addedFields) {
- relBuilder.project(originalExtraNodes);
- }
- return;
- }
-
- private TrimResult result(RelNode r, final Mapping mapping) {
- final RexBuilder rexBuilder = relBuilder.getRexBuilder();
- for (final CorrelationId correlation : r.getVariablesSet()) {
- r = r.accept(
- new CorrelationReferenceFinder() {
- @Override
- protected RexNode handle(RexFieldAccess fieldAccess) {
- final RexCorrelVariable v =
- (RexCorrelVariable) fieldAccess.getReferenceExpr();
- if (v.id.equals(correlation)
- && v.getType().getFieldCount() == mapping.getSourceCount()) {
- final int old = fieldAccess.getField().getIndex();
- final int new_ = mapping.getTarget(old);
- final RelDataTypeFactory.FieldInfoBuilder typeBuilder =
- relBuilder.getTypeFactory().builder();
- for (int target : Util.range(mapping.getTargetCount())) {
- typeBuilder.add(
- v.getType().getFieldList().get(mapping.getSource(target)));
- }
- final RexNode newV =
- rexBuilder.makeCorrel(typeBuilder.build(), v.id);
- if (old != new_) {
- return rexBuilder.makeFieldAccess(newV, new_);
- }
- }
- return fieldAccess;
- }
-
- });
+ // Project nulls for the extra fields. (Maybe a sub-class table has
+ // extra fields, but we don't.)
+ for (RelDataTypeField extraField : extraFields) {
+ exprList.add(
+ rexBuilder.ensureType(
+ extraField.getType(),
+ rexBuilder.constantNull(),
+ true));
+ nameList.add(extraField.getName());
}
- return new TrimResult(r, mapping);
+
+ HiveProject hp = (HiveProject) relBuilder.push(dq).project(exprList, nameList).build();
+ hp.setSynthetic();
+ return hp;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java
index 18fe650..b86b947 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdCollation.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.stats;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.metadata.BuiltInMetadata;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
+import org.apache.calcite.rel.metadata.MetadataDef;
+import org.apache.calcite.rel.metadata.MetadataHandler;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdCollation;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -32,7 +35,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import com.google.common.collect.ImmutableList;
-public class HiveRelMdCollation {
+public class HiveRelMdCollation implements MetadataHandler<BuiltInMetadata.Collation> {
public static final RelMetadataProvider SOURCE =
ChainedRelMetadataProvider.of(
@@ -47,6 +50,11 @@ public class HiveRelMdCollation {
//~ Methods ----------------------------------------------------------------
+ @Override
+ public MetadataDef<BuiltInMetadata.Collation> getDef() {
+ return BuiltInMetadata.Collation.DEF;
+ }
+
public ImmutableList<RelCollation> collations(HiveAggregate aggregate, RelMetadataQuery mq) {
// Compute collations
ImmutableList.Builder<RelFieldCollation> collationListBuilder =
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java
index 62d3ead..e574e7a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistribution.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.stats;
import org.apache.calcite.rel.RelDistribution;
+import org.apache.calcite.rel.metadata.BuiltInMetadata;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
+import org.apache.calcite.rel.metadata.MetadataDef;
+import org.apache.calcite.rel.metadata.MetadataHandler;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdDistribution;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.util.BuiltInMethod;
@@ -30,7 +32,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import com.google.common.collect.ImmutableList;
-public class HiveRelMdDistribution {
+public class HiveRelMdDistribution implements MetadataHandler<BuiltInMetadata.Distribution> {
public static final RelMetadataProvider SOURCE =
ChainedRelMetadataProvider.of(
@@ -44,6 +46,10 @@ public class HiveRelMdDistribution {
//~ Methods ----------------------------------------------------------------
+ public MetadataDef<BuiltInMetadata.Distribution> getDef() {
+ return BuiltInMetadata.Distribution.DEF;
+ }
+
public RelDistribution distribution(HiveAggregate aggregate, RelMetadataQuery mq) {
return new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED,
aggregate.getGroupSet().asList());
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
index e468573..69e157e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
@@ -40,6 +40,10 @@ import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.SemiJoin;
import org.apache.calcite.rel.core.Union;
+import org.apache.calcite.rel.metadata.BuiltInMetadata;
+import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
+import org.apache.calcite.rel.metadata.MetadataDef;
+import org.apache.calcite.rel.metadata.MetadataHandler;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdPredicates;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -72,14 +76,28 @@ import com.google.common.collect.Maps;
//TODO: Move this to calcite
-public class HiveRelMdPredicates extends RelMdPredicates {
+public class HiveRelMdPredicates implements MetadataHandler<BuiltInMetadata.Predicates> {
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource(
- BuiltInMethod.PREDICATES.method,
- new HiveRelMdPredicates());
+ public static final RelMetadataProvider SOURCE =
+ ChainedRelMetadataProvider.of(
+ ImmutableList.of(
+ ReflectiveRelMetadataProvider.reflectiveSource(
+ BuiltInMethod.PREDICATES.method, new HiveRelMdPredicates()),
+ RelMdPredicates.SOURCE));
private static final List<RexNode> EMPTY_LIST = ImmutableList.of();
+ //~ Constructors -----------------------------------------------------------
+
+ private HiveRelMdPredicates() {}
+
+ //~ Methods ----------------------------------------------------------------
+
+ @Override
+ public MetadataDef<BuiltInMetadata.Predicates> getDef() {
+ return BuiltInMetadata.Predicates.DEF;
+ }
+
/**
* Infers predicates for a project.
*
@@ -99,8 +117,8 @@ public class HiveRelMdPredicates extends RelMdPredicates {
*
* </ol>
*/
- @Override
public RelOptPredicateList getPredicates(Project project, RelMetadataQuery mq) {
+
RelNode child = project.getInput();
final RexBuilder rexBuilder = project.getCluster().getRexBuilder();
RelOptPredicateList childInfo = mq.getPulledUpPredicates(child);
@@ -151,7 +169,6 @@ public class HiveRelMdPredicates extends RelMdPredicates {
}
/** Infers predicates for a {@link org.apache.calcite.rel.core.Join}. */
- @Override
public RelOptPredicateList getPredicates(Join join, RelMetadataQuery mq) {
RexBuilder rB = join.getCluster().getRexBuilder();
RelNode left = join.getInput(0);
@@ -181,7 +198,6 @@ public class HiveRelMdPredicates extends RelMdPredicates {
* pulledUpExprs : { a > 7}
* </pre>
*/
- @Override
public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq) {
final RelNode input = agg.getInput();
final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input);
@@ -209,7 +225,6 @@ public class HiveRelMdPredicates extends RelMdPredicates {
/**
* Infers predicates for a Union.
*/
- @Override
public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) {
RexBuilder rB = union.getCluster().getRexBuilder();
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
index 0d03ebb..651adc0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
@@ -41,13 +41,16 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import com.google.common.collect.ImmutableMap;
public class HiveRelMdSelectivity extends RelMdSelectivity {
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource(
- BuiltInMethod.SELECTIVITY.method,
- new HiveRelMdSelectivity());
- protected HiveRelMdSelectivity() {
- super();
- }
+ public static final RelMetadataProvider SOURCE =
+ ReflectiveRelMetadataProvider.reflectiveSource(
+ BuiltInMethod.SELECTIVITY.method, new HiveRelMdSelectivity());
+
+ //~ Constructors -----------------------------------------------------------
+
+ private HiveRelMdSelectivity() {}
+
+ //~ Methods ----------------------------------------------------------------
public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) {
if (predicate != null) {
@@ -58,7 +61,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
return 1.0;
}
- public Double getSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) throws CalciteSemanticException {
+ public Double getSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) {
if (j.getJoinType().equals(JoinRelType.INNER)) {
return computeInnerJoinSelectivity(j, mq, predicate);
} else if (j.getJoinType().equals(JoinRelType.LEFT) ||
@@ -75,7 +78,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
return 1.0;
}
- private Double computeInnerJoinSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) throws CalciteSemanticException {
+ private Double computeInnerJoinSelectivity(HiveJoin j, RelMetadataQuery mq, RexNode predicate) {
double ndvCrossProduct = 1;
Pair<Boolean, RexNode> predInfo =
getCombinedPredicateForJoin(j, predicate);
@@ -86,8 +89,13 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
}
RexNode combinedPredicate = predInfo.getValue();
- JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j,
- combinedPredicate);
+ JoinPredicateInfo jpi;
+ try {
+ jpi = JoinPredicateInfo.constructJoinPredicateInfo(j,
+ combinedPredicate);
+ } catch (CalciteSemanticException e) {
+ throw new RuntimeException(e);
+ }
ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap
.builder();
ImmutableMap<Integer, Double> colStatMap;
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
index 31adb41..1039f56 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
@@ -122,7 +122,18 @@ public class HiveRelMdSize extends RelMdSize {
case BIGINT:
case DOUBLE:
case TIMESTAMP:
- case INTERVAL_DAY_TIME:
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_MONTH:
+ case INTERVAL_SECOND:
+ case INTERVAL_YEAR:
case INTERVAL_YEAR_MONTH:
return 8d;
case BINARY:
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
index 0718150..9a5a2ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
@@ -30,9 +30,9 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.BuiltInMetadata;
-import org.apache.calcite.rel.metadata.Metadata;
+import org.apache.calcite.rel.metadata.MetadataDef;
+import org.apache.calcite.rel.metadata.MetadataHandler;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdUniqueKeys;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexInputRef;
@@ -43,13 +43,16 @@ import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
-import com.google.common.base.Function;
+public class HiveRelMdUniqueKeys implements MetadataHandler<BuiltInMetadata.UniqueKeys> {
-public class HiveRelMdUniqueKeys {
+ public static final RelMetadataProvider SOURCE =
+ ReflectiveRelMetadataProvider.reflectiveSource(
+ BuiltInMethod.UNIQUE_KEYS.method, new HiveRelMdUniqueKeys());
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider
- .reflectiveSource(BuiltInMethod.UNIQUE_KEYS.method,
- new HiveRelMdUniqueKeys());
+ @Override
+ public MetadataDef<BuiltInMetadata.UniqueKeys> getDef() {
+ return BuiltInMetadata.UniqueKeys.DEF;
+ }
/*
* Infer Uniquenes if: - rowCount(col) = ndv(col) - TBD for numerics: max(col)
@@ -65,7 +68,60 @@ public class HiveRelMdUniqueKeys {
HiveTableScan tScan = getTableScan(rel.getInput(), false);
if (tScan == null) {
- return mq.getUniqueKeys(rel, ignoreNulls);
+ // If HiveTableScan is not found, e.g., not sequence of Project and
+ // Filter operators, execute the original getUniqueKeys method
+
+ // LogicalProject maps a set of rows to a different set;
+ // Without knowledge of the mapping function(whether it
+ // preserves uniqueness), it is only safe to derive uniqueness
+ // info from the child of a project when the mapping is f(a) => a.
+ //
+ // Further more, the unique bitset coming from the child needs
+ // to be mapped to match the output of the project.
+ final Map<Integer, Integer> mapInToOutPos = new HashMap<>();
+ final List<RexNode> projExprs = rel.getProjects();
+ final Set<ImmutableBitSet> projUniqueKeySet = new HashSet<>();
+
+ // Build an input to output position map.
+ for (int i = 0; i < projExprs.size(); i++) {
+ RexNode projExpr = projExprs.get(i);
+ if (projExpr instanceof RexInputRef) {
+ mapInToOutPos.put(((RexInputRef) projExpr).getIndex(), i);
+ }
+ }
+
+ if (mapInToOutPos.isEmpty()) {
+ // if there's no RexInputRef in the projected expressions
+ // return empty set.
+ return projUniqueKeySet;
+ }
+
+ Set<ImmutableBitSet> childUniqueKeySet =
+ mq.getUniqueKeys(rel.getInput(), ignoreNulls);
+
+ if (childUniqueKeySet != null) {
+ // Now add to the projUniqueKeySet the child keys that are fully
+ // projected.
+ for (ImmutableBitSet colMask : childUniqueKeySet) {
+ ImmutableBitSet.Builder tmpMask = ImmutableBitSet.builder();
+ boolean completeKeyProjected = true;
+ for (int bit : colMask) {
+ if (mapInToOutPos.containsKey(bit)) {
+ tmpMask.set(mapInToOutPos.get(bit));
+ } else {
+ // Skip the child unique key if part of it is not
+ // projected.
+ completeKeyProjected = false;
+ break;
+ }
+ }
+ if (completeKeyProjected) {
+ projUniqueKeySet.add(tmpMask.build());
+ }
+ }
+ }
+
+ return projUniqueKeySet;
}
Map<Integer, Integer> posMap = new HashMap<Integer, Integer>();
http://git-wip-us.apache.org/repos/asf/hive/blob/b597ab2a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
index 9a5becb..7b9ee84 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
@@ -22,15 +22,15 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import org.apache.calcite.adapter.druid.DruidQuery;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
-import org.apache.hadoop.hive.ql.optimizer.calcite.druid.DruidQuery;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
@@ -58,19 +58,20 @@ class ASTBuilder {
"TOK_TMP_FILE")).node();
}
- static ASTNode table(TableScan scan) {
- RelOptHiveTable hTbl = (RelOptHiveTable) scan.getTable();
- ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add(
- ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME")
- .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName())
- .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName()));
-
+ static ASTNode table(RelNode scan) {
HiveTableScan hts;
if (scan instanceof DruidQuery) {
hts = (HiveTableScan) ((DruidQuery)scan).getTableScan();
} else {
hts = (HiveTableScan) scan;
}
+
+ RelOptHiveTable hTbl = (RelOptHiveTable) hts.getTable();
+ ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_TABREF, "TOK_TABREF").add(
+ ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME")
+ .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName())
+ .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName()));
+
ASTBuilder propList = ASTBuilder.construct(HiveParser.TOK_TABLEPROPLIST, "TOK_TABLEPROPLIST");
if (scan instanceof DruidQuery) {
// Pass possible query to Druid
@@ -181,8 +182,19 @@ class ASTBuilder {
case DATE:
case TIME:
case TIMESTAMP:
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_MONTH:
+ case INTERVAL_SECOND:
+ case INTERVAL_YEAR:
case INTERVAL_YEAR_MONTH:
- case INTERVAL_DAY_TIME:
if (literal.getValue() == null) {
return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node();
}
@@ -273,14 +285,25 @@ class ASTBuilder {
val = "'" + val + "'";
}
break;
+ case INTERVAL_YEAR:
+ case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH: {
type = HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL;
BigDecimal monthsBd = (BigDecimal) literal.getValue();
HiveIntervalYearMonth intervalYearMonth = new HiveIntervalYearMonth(monthsBd.intValue());
val = "'" + intervalYearMonth.toString() + "'";
- break;
}
- case INTERVAL_DAY_TIME: {
+ break;
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_SECOND: {
type = HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL;
BigDecimal millisBd = (BigDecimal) literal.getValue();
@@ -288,8 +311,8 @@ class ASTBuilder {
BigDecimal secsBd = millisBd.divide(BigDecimal.valueOf(1000));
HiveIntervalDayTime intervalDayTime = new HiveIntervalDayTime(secsBd);
val = "'" + intervalDayTime.toString() + "'";
- break;
}
+ break;
case NULL:
type = HiveParser.TOK_NULL;
break;