You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2015/12/23 06:06:52 UTC
[9/9] hive git commit: HIVE-11927: Implement/Enable constant related
optimization rules in Calcite: enable HiveReduceExpressionsRule to fold
constants (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
HIVE-11927: Implement/Enable constant related optimization rules in Calcite: enable HiveReduceExpressionsRule to fold constants (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b340ecb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b340ecb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b340ecb5
Branch: refs/heads/master
Commit: b340ecb5e163277e86def59b454b8c041ece39d5
Parents: af96223
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Dec 22 21:06:14 2015 -0800
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Tue Dec 22 21:06:14 2015 -0800
----------------------------------------------------------------------
.../calcite/CalciteSemanticException.java | 2 +-
.../optimizer/calcite/HiveRexExecutorImpl.java | 80 ++
.../rules/HiveReduceExpressionsRule.java | 905 +++++++++++++++++++
.../calcite/translator/ASTBuilder.java | 30 +-
.../calcite/translator/RexNodeConverter.java | 48 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 84 +-
ql/src/test/queries/clientpositive/cbo_const.q | 52 ++
.../queries/clientpositive/constantfolding.q | 88 ++
.../clientpositive/annotate_stats_select.q.out | 58 +-
.../bucketizedhiveinputformat.q.out | 2 +
ql/src/test/results/clientpositive/cast1.q.out | 10 +-
.../test/results/clientpositive/cbo_const.q.out | 334 +++++++
.../cbo_rp_cross_product_check_2.q.out | 4 +-
.../clientpositive/cbo_rp_lineage2.q.out | 2 +-
.../clientpositive/constantfolding.q.out | 305 +++++++
.../clientpositive/cross_product_check_1.q.out | 4 +-
.../clientpositive/cross_product_check_2.q.out | 4 +-
.../clientpositive/dynamic_rdd_cache.q.out | 46 +-
.../dynpart_sort_optimization2.q.out | 45 +-
.../results/clientpositive/groupby_ppd.q.out | 18 +-
.../clientpositive/groupby_sort_1_23.q.out | 12 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 12 +-
.../results/clientpositive/input_part1.q.out | 4 +-
.../results/clientpositive/input_part5.q.out | 2 +-
.../results/clientpositive/input_part6.q.out | 2 +-
.../test/results/clientpositive/lineage2.q.out | 2 +-
.../test/results/clientpositive/lineage3.q.out | 10 +-
.../list_bucket_query_oneskew_2.q.out | 50 +-
.../llap/bucket_map_join_tez1.q.out | 4 +-
.../llap/vector_join_part_col_char.q.out | 2 +-
.../clientpositive/orc_predicate_pushdown.q.out | 22 +-
.../parquet_predicate_pushdown.q.out | 22 +-
.../clientpositive/partition_multilevels.q.out | 44 +-
.../results/clientpositive/perf/query31.q.out | 338 +++----
.../results/clientpositive/perf/query39.q.out | 58 +-
.../results/clientpositive/perf/query42.q.out | 171 ++--
.../results/clientpositive/perf/query52.q.out | 163 ++--
.../results/clientpositive/perf/query64.q.out | 362 ++++----
.../results/clientpositive/perf/query66.q.out | 627 ++++++-------
.../results/clientpositive/perf/query75.q.out | 643 ++++++-------
.../results/clientpositive/pointlookup2.q.out | 6 +-
.../results/clientpositive/quotedid_basic.q.out | 62 +-
.../clientpositive/quotedid_partition.q.out | 22 +-
.../spark/bucket_map_join_tez1.q.out | 4 +-
.../spark/bucketizedhiveinputformat.q.out | 2 +
.../spark/cross_product_check_1.q.out | 4 +-
.../spark/cross_product_check_2.q.out | 4 +-
.../spark/dynamic_rdd_cache.q.out | 46 +-
.../spark/groupby_sort_1_23.q.out | 12 +-
.../spark/groupby_sort_skew_1_23.q.out | 12 +-
.../clientpositive/spark/union_remove_25.q.out | 16 +-
.../clientpositive/spark/union_view.q.out | 60 +-
.../results/clientpositive/subquery_notin.q.out | 18 +-
.../subquery_notin_having.q.java1.7.out | 10 +-
.../subquery_unqualcolumnrefs.q.out | 2 +-
.../tez/bucket_map_join_tez1.q.out | 4 +-
.../tez/cross_product_check_1.q.out | 4 +-
.../tez/cross_product_check_2.q.out | 4 +-
.../tez/dynpart_sort_optimization2.q.out | 34 +-
.../clientpositive/tez/explainuser_1.q.out | 260 +++---
.../tez/vector_decimal_round.q.out | 6 +-
ql/src/test/results/clientpositive/udf1.q.out | 40 +-
.../results/clientpositive/udf_10_trims.q.out | 2 +-
.../clientpositive/udf_folder_constants.q.out | 8 +-
.../clientpositive/union_remove_25.q.out | 20 +-
.../results/clientpositive/union_view.q.out | 60 +-
.../clientpositive/vector_decimal_round.q.out | 6 +-
67 files changed, 3660 insertions(+), 1739 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
index 336745b..f6b4124 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
@@ -31,7 +31,7 @@ public class CalciteSemanticException extends SemanticException {
public enum UnsupportedFeature {
Distinct_without_an_aggreggation, Duplicates_in_RR, Filter_expression_with_non_boolean_return_type,
- Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal,
+ Having_clause_without_any_groupby, Hint, Invalid_column_reference, Invalid_decimal, Invalid_interval,
Less_than_equal_greater_than, Multi_insert, Others, Same_name_in_multiple_expressions,
Schema_less_table, Select_alias_in_having_clause, Select_transform, Subquery,
Table_sample_clauses, UDTF, Union_type, Unique_join
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
new file mode 100644
index 0000000..f7958c6
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.calcite;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+public class HiveRexExecutorImpl implements RelOptPlanner.Executor {
+
+ private final RelOptCluster cluster;
+
+ protected final Logger LOG;
+
+ public HiveRexExecutorImpl(RelOptCluster cluster) {
+ this.cluster = cluster;
+ LOG = LoggerFactory.getLogger(this.getClass().getName());
+ }
+
+ @Override
+ public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) {
+ RexNodeConverter rexNodeConverter = new RexNodeConverter(cluster);
+ for (RexNode rexNode : constExps) {
+ // initialize the converter
+ ExprNodeConverter converter = new ExprNodeConverter("", null, null, null,
+ new HashSet<Integer>(), cluster.getTypeFactory());
+ // convert RexNode to ExprNodeGenericFuncDesc
+ ExprNodeDesc expr = rexNode.accept(converter);
+ if (expr instanceof ExprNodeGenericFuncDesc) {
+ // folding the constant
+ ExprNodeDesc constant = ConstantPropagateProcFactory
+ .foldExpr((ExprNodeGenericFuncDesc) expr);
+ if (constant != null) {
+ try {
+ // convert constant back to RexNode
+ reducedValues.add(rexNodeConverter.convert((ExprNodeConstantDesc) constant));
+ } catch (Exception e) {
+ LOG.warn(e.getMessage());
+ reducedValues.add(rexNode);
+ }
+ } else {
+ reducedValues.add(rexNode);
+ }
+ } else {
+ reducedValues.add(rexNode);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
new file mode 100644
index 0000000..50e139b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptPredicateList;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexShuttle;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlRowOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Stacks;
+import org.apache.calcite.util.Util;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Collection of planner rules that apply various simplifying transformations on
+ * RexNode trees. Currently, there are two transformations:
+ *
+ * <ul>
+ * <li>Constant reduction, which evaluates constant subtrees, replacing them
+ * with a corresponding RexLiteral
+ * <li>Removal of redundant casts, which occurs when the argument into the cast
+ * is the same as the type of the resulting cast expression
+ * </ul>
+ */
+public abstract class HiveReduceExpressionsRule extends RelOptRule {
+ //~ Static fields/initializers ---------------------------------------------
+
+ /**
+ * Regular expression that matches the description of all instances of this
+ * rule and {@link ValuesReduceRule} also. Use
+ * it to prevent the planner from invoking these rules.
+ */
+ public static final Pattern EXCLUSION_PATTERN =
+ Pattern.compile("Reduce(Expressions|Values)Rule.*");
+
+ /**
+ * Singleton rule that reduces constants inside a
+ * {@link org.apache.calcite.rel.logical.HiveFilter}.
+ */
+ public static final HiveReduceExpressionsRule FILTER_INSTANCE =
+ new FilterReduceExpressionsRule(HiveFilter.class, HiveRelFactories.HIVE_BUILDER);
+
+ /**
+ * Singleton rule that reduces constants inside a
+ * {@link org.apache.calcite.rel.logical.HiveProject}.
+ */
+ public static final HiveReduceExpressionsRule PROJECT_INSTANCE =
+ new ProjectReduceExpressionsRule(HiveProject.class, HiveRelFactories.HIVE_BUILDER);
+
+ /**
+ * Singleton rule that reduces constants inside a
+ * {@link org.apache.calcite.rel.core.HiveJoin}.
+ */
+ public static final HiveReduceExpressionsRule JOIN_INSTANCE =
+ new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER);
+
+ /**
+ * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Filter}.
+ * If the condition is a constant, the filter is removed (if TRUE) or replaced with
+ * an empty {@link org.apache.calcite.rel.core.Values} (if FALSE or NULL).
+ */
+ public static class FilterReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+ public FilterReduceExpressionsRule(Class<? extends Filter> filterClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(filterClass, relBuilderFactory, "HiveReduceExpressionsRule(Filter)");
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+ final List<RexNode> expList =
+ Lists.newArrayList(filter.getCondition());
+ RexNode newConditionExp;
+ boolean reduced;
+ final RelOptPredicateList predicates =
+ RelMetadataQuery.getPulledUpPredicates(filter.getInput());
+ if (reduceExpressions(filter, expList, predicates)) {
+ assert expList.size() == 1;
+ newConditionExp = expList.get(0);
+ reduced = true;
+ } else {
+ // No reduction, but let's still test the original
+ // predicate to see if it was already a constant,
+ // in which case we don't need any runtime decision
+ // about filtering.
+ newConditionExp = filter.getCondition();
+ reduced = false;
+ }
+ if (newConditionExp.isAlwaysTrue()) {
+ call.transformTo(
+ filter.getInput());
+ }
+ // TODO: support LogicalValues
+ else if (newConditionExp instanceof RexLiteral
+ || RexUtil.isNullLiteral(newConditionExp, true)) {
+ // call.transformTo(call.builder().values(filter.getRowType()).build());
+ return;
+ }
+ else if (reduced) {
+ call.transformTo(call.builder().
+ push(filter.getInput()).filter(expList.get(0)).build());
+ } else {
+ if (newConditionExp instanceof RexCall) {
+ RexCall rexCall = (RexCall) newConditionExp;
+ boolean reverse =
+ rexCall.getOperator()
+ == SqlStdOperatorTable.NOT;
+ if (reverse) {
+ rexCall = (RexCall) rexCall.getOperands().get(0);
+ }
+ reduceNotNullableFilter(call, filter, rexCall, reverse);
+ }
+ return;
+ }
+
+ // New plan is absolutely better than old plan.
+ call.getPlanner().setImportance(filter, 0.0);
+ }
+
+ private void reduceNotNullableFilter(
+ RelOptRuleCall call,
+ Filter filter,
+ RexCall rexCall,
+ boolean reverse) {
+ // If the expression is a IS [NOT] NULL on a non-nullable
+ // column, then we can either remove the filter or replace
+ // it with an Empty.
+ boolean alwaysTrue;
+ switch (rexCall.getKind()) {
+ case IS_NULL:
+ case IS_UNKNOWN:
+ alwaysTrue = false;
+ break;
+ case IS_NOT_NULL:
+ alwaysTrue = true;
+ break;
+ default:
+ return;
+ }
+ if (reverse) {
+ alwaysTrue = !alwaysTrue;
+ }
+ RexNode operand = rexCall.getOperands().get(0);
+ if (operand instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) operand;
+ if (!inputRef.getType().isNullable()) {
+ if (alwaysTrue) {
+ call.transformTo(filter.getInput());
+ } else {
+ // TODO: support LogicalValues
+ // call.transformTo(call.builder().values(filter.getRowType()).build());
+ return;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.Project}.
+ */
+ public static class ProjectReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+ public ProjectReduceExpressionsRule(Class<? extends Project> projectClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(projectClass, relBuilderFactory, "HiveReduceExpressionsRule(Project)");
+ }
+
+ public boolean matches(RelOptRuleCall call) {
+ Project project = call.rel(0);
+ HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+
+ // If this operator has been visited already by the rule,
+ // we do not need to apply the optimization
+ if (registry != null && registry.getVisited(this).contains(project)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ Project project = call.rel(0);
+ // Register that we have visited this operator in this rule
+ HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+ if (registry != null) {
+ registry.registerVisited(this, project);
+ }
+ final RelOptPredicateList predicates =
+ RelMetadataQuery.getPulledUpPredicates(project.getInput());
+ final List<RexNode> expList =
+ Lists.newArrayList(project.getProjects());
+ if (reduceExpressions(project, expList, predicates)) {
+ RelNode newProject = call.builder().push(project.getInput())
+ .project(expList, project.getRowType().getFieldNames()).build();
+ if (registry != null) {
+ registry.registerVisited(this, newProject);
+ }
+ call.transformTo(newProject);
+
+ // New plan is absolutely better than old plan.
+ call.getPlanner().setImportance(project, 0.0);
+ }
+ }
+ }
+
+ /**
+ * Rule that reduces constants inside a {@link org.apache.calcite.rel.core.HiveJoin}.
+ */
+ public static class JoinReduceExpressionsRule extends HiveReduceExpressionsRule {
+
+ public JoinReduceExpressionsRule(Class<? extends HiveJoin> joinClass,
+ RelBuilderFactory relBuilderFactory) {
+ super(joinClass, relBuilderFactory, "HiveReduceExpressionsRule(HiveJoin)");
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final HiveJoin join = call.rel(0);
+ final List<RexNode> expList = Lists.newArrayList(join.getCondition());
+ final int fieldCount = join.getLeft().getRowType().getFieldCount();
+ final RelOptPredicateList leftPredicates =
+ RelMetadataQuery.getPulledUpPredicates(join.getLeft());
+ final RelOptPredicateList rightPredicates =
+ RelMetadataQuery.getPulledUpPredicates(join.getRight());
+ final RelOptPredicateList predicates =
+ leftPredicates.union(rightPredicates.shift(fieldCount));
+ if (!reduceExpressions(join, expList, predicates)) {
+ return;
+ }
+ final JoinInfo joinInfo = JoinInfo.of(join.getLeft(), join.getRight(), expList.get(0));
+ if (!joinInfo.isEqui()) {
+ // This kind of join must be an equi-join, and the condition is
+ // no longer an equi-join. SemiJoin is an example of this.
+ return;
+ }
+ call.transformTo(
+ join.copy(
+ join.getTraitSet(),
+ expList.get(0),
+ join.getLeft(),
+ join.getRight(),
+ join.getJoinType(),
+ join.isSemiJoinDone()));
+
+ // New plan is absolutely better than old plan.
+ call.getPlanner().setImportance(join, 0.0);
+ }
+ }
+
+ //~ Constructors -----------------------------------------------------------
+
+ /**
+ * Creates a HiveReduceExpressionsRule.
+ *
+ * @param clazz class of rels to which this rule should apply
+ */
+ protected HiveReduceExpressionsRule(Class<? extends RelNode> clazz,
+ RelBuilderFactory relBuilderFactory, String desc) {
+ super(operand(clazz, any()), relBuilderFactory, desc);
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ /**
+ * Reduces a list of expressions.
+ *
+ * @param rel Relational expression
+ * @param expList List of expressions, modified in place
+ * @param predicates Constraints known to hold on input expressions
+ * @return whether reduction found something to change, and succeeded
+ */
+ protected static boolean reduceExpressions(RelNode rel, List<RexNode> expList,
+ RelOptPredicateList predicates) {
+ RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
+
+ // Replace predicates on CASE to CASE on predicates.
+ new CaseShuttle().mutate(expList);
+
+ // Find reducible expressions.
+ final List<RexNode> constExps = Lists.newArrayList();
+ List<Boolean> addCasts = Lists.newArrayList();
+ final List<RexNode> removableCasts = Lists.newArrayList();
+ final ImmutableMap<RexNode, RexLiteral> constants =
+ predicateConstants(predicates);
+ findReducibleExps(rel.getCluster().getTypeFactory(), expList, constants,
+ constExps, addCasts, removableCasts);
+ if (constExps.isEmpty() && removableCasts.isEmpty()) {
+ return false;
+ }
+
+ // Remove redundant casts before reducing constant expressions.
+ // If the argument to the redundant cast is a reducible constant,
+ // reducing that argument to a constant first will result in not being
+ // able to locate the original cast expression.
+ if (!removableCasts.isEmpty()) {
+ final List<RexNode> reducedExprs = Lists.newArrayList();
+ for (RexNode exp : removableCasts) {
+ RexCall call = (RexCall) exp;
+ reducedExprs.add(call.getOperands().get(0));
+ }
+ RexReplacer replacer =
+ new RexReplacer(
+ rexBuilder,
+ removableCasts,
+ reducedExprs,
+ Collections.nCopies(removableCasts.size(), false));
+ replacer.mutate(expList);
+ }
+
+ if (constExps.isEmpty()) {
+ return true;
+ }
+
+ final List<RexNode> constExps2 = Lists.newArrayList(constExps);
+ if (!constants.isEmpty()) {
+ //noinspection unchecked
+ final List<Map.Entry<RexNode, RexNode>> pairs =
+ (List<Map.Entry<RexNode, RexNode>>) (List)
+ Lists.newArrayList(constants.entrySet());
+ RexReplacer replacer =
+ new RexReplacer(
+ rexBuilder,
+ Pair.left(pairs),
+ Pair.right(pairs),
+ Collections.nCopies(pairs.size(), false));
+ replacer.mutate(constExps2);
+ }
+
+ // Compute the values they reduce to.
+ RelOptPlanner.Executor executor =
+ rel.getCluster().getPlanner().getExecutor();
+ if (executor == null) {
+ // Cannot reduce expressions: caller has not set an executor in their
+ // environment. Caller should execute something like the following before
+ // invoking the planner:
+ //
+ // final RexExecutorImpl executor =
+ // new RexExecutorImpl(Schemas.createDataContext(null));
+ // rootRel.getCluster().getPlanner().setExecutor(executor);
+ return false;
+ }
+
+ final List<RexNode> reducedValues = Lists.newArrayList();
+ executor.reduce(rexBuilder, constExps2, reducedValues);
+
+ // For Project, we have to be sure to preserve the result
+ // types, so always cast regardless of the expression type.
+ // For other RelNodes like Filter, in general, this isn't necessary,
+ // and the presence of casts could hinder other rules such as sarg
+ // analysis, which require bare literals. But there are special cases,
+ // like when the expression is a UDR argument, that need to be
+ // handled as special cases.
+ if (rel instanceof Project) {
+ addCasts = Collections.nCopies(reducedValues.size(), true);
+ }
+
+ RexReplacer replacer =
+ new RexReplacer(
+ rexBuilder,
+ constExps,
+ reducedValues,
+ addCasts);
+ replacer.mutate(expList);
+ return true;
+ }
+
+ /**
+ * Locates expressions that can be reduced to literals or converted to
+ * expressions with redundant casts removed.
+ *
+ * @param typeFactory Type factory
+ * @param exps list of candidate expressions to be examined for
+ * reduction
+ * @param constants List of expressions known to be constant
+ * @param constExps returns the list of expressions that can be constant
+ * reduced
+ * @param addCasts indicator for each expression that can be constant
+ * reduced, whether a cast of the resulting reduced
+ * expression is potentially necessary
+ * @param removableCasts returns the list of cast expressions where the cast
+ */
+ protected static void findReducibleExps(RelDataTypeFactory typeFactory,
+ List<RexNode> exps, ImmutableMap<RexNode, RexLiteral> constants,
+ List<RexNode> constExps, List<Boolean> addCasts,
+ List<RexNode> removableCasts) {
+ ReducibleExprLocator gardener =
+ new ReducibleExprLocator(typeFactory, constants, constExps,
+ addCasts, removableCasts);
+ for (RexNode exp : exps) {
+ gardener.analyze(exp);
+ }
+ assert constExps.size() == addCasts.size();
+ }
+
+ protected static ImmutableMap<RexNode, RexLiteral> predicateConstants(
+ RelOptPredicateList predicates) {
+ // We cannot use an ImmutableMap.Builder here. If there are multiple entries
+ // with the same key (e.g. "WHERE deptno = 1 AND deptno = 2"), it doesn't
+ // matter which we take, so the latter will replace the former.
+ // The basic idea is to find all the pairs of RexNode = RexLiteral
+ // (1) If 'predicates' contain a non-EQUALS, we bail out.
+ // (2) It is OK if a RexNode is equal to the same RexLiteral several times,
+ // (e.g. "WHERE deptno = 1 AND deptno = 1")
+ // (3) It will return false if there are inconsistent constraints (e.g.
+ // "WHERE deptno = 1 AND deptno = 2")
+ final Map<RexNode, RexLiteral> map = new HashMap<>();
+ final Set<RexNode> excludeSet = new HashSet<>();
+ for (RexNode predicate : predicates.pulledUpPredicates) {
+ gatherConstraints(map, predicate, excludeSet);
+ }
+ final ImmutableMap.Builder<RexNode, RexLiteral> builder =
+ ImmutableMap.builder();
+ for (Map.Entry<RexNode, RexLiteral> entry : map.entrySet()) {
+ RexNode rexNode = entry.getKey();
+ if (!overlap(rexNode, excludeSet)) {
+ builder.put(rexNode, entry.getValue());
+ }
+ }
+ return builder.build();
+ }
+
+ private static boolean overlap(RexNode rexNode, Set<RexNode> set) {
+ if (rexNode instanceof RexCall) {
+ for (RexNode r : ((RexCall) rexNode).getOperands()) {
+ if (overlap(r, set)) {
+ return true;
+ }
+ }
+ return false;
+ } else {
+ return set.contains(rexNode);
+ }
+ }
+
+ /** Tries to decompose the RexNode which is a RexCall into non-literal
+ * RexNodes. */
+ private static void decompose(Set<RexNode> set, RexNode rexNode) {
+ if (rexNode instanceof RexCall) {
+ for (RexNode r : ((RexCall) rexNode).getOperands()) {
+ decompose(set, r);
+ }
+ } else if (!(rexNode instanceof RexLiteral)) {
+ set.add(rexNode);
+ }
+ }
+
+ private static void gatherConstraints(Map<RexNode, RexLiteral> map,
+ RexNode predicate, Set<RexNode> excludeSet) {
+ if (predicate.getKind() != SqlKind.EQUALS) {
+ decompose(excludeSet, predicate);
+ return;
+ }
+ final List<RexNode> operands = ((RexCall) predicate).getOperands();
+ if (operands.size() != 2) {
+ decompose(excludeSet, predicate);
+ return;
+ }
+ // if it reaches here, we have rexNode equals rexNode
+ final RexNode left = operands.get(0);
+ final RexNode right = operands.get(1);
+ // note that literals are immutable too and they can only be compared through
+ // values.
+ if (right instanceof RexLiteral && !excludeSet.contains(left)) {
+ RexLiteral existedValue = map.get(left);
+ if (existedValue == null) {
+ map.put(left, (RexLiteral) right);
+ } else {
+ if (!existedValue.getValue().equals(((RexLiteral) right).getValue())) {
+ // we found conflict values.
+ map.remove(left);
+ excludeSet.add(left);
+ }
+ }
+ } else if (left instanceof RexLiteral && !excludeSet.contains(right)) {
+ RexLiteral existedValue = map.get(right);
+ if (existedValue == null) {
+ map.put(right, (RexLiteral) left);
+ } else {
+ if (!existedValue.getValue().equals(((RexLiteral) left).getValue())) {
+ map.remove(right);
+ excludeSet.add(right);
+ }
+ }
+ }
+ }
+
+ /** Pushes predicates into a CASE.
+ *
+ * <p>We have a loose definition of 'predicate': any boolean expression will
+ * do, except CASE. For example '(CASE ...) = 5' or '(CASE ...) IS NULL'.
+ */
+ protected static RexCall pushPredicateIntoCase(RexCall call) {
+ if (call.getType().getSqlTypeName() != SqlTypeName.BOOLEAN) {
+ return call;
+ }
+ switch (call.getKind()) {
+ case CASE:
+ case AND:
+ case OR:
+ return call; // don't push CASE into CASE!
+ }
+ int caseOrdinal = -1;
+ final List<RexNode> operands = call.getOperands();
+ for (int i = 0; i < operands.size(); i++) {
+ RexNode operand = operands.get(i);
+ switch (operand.getKind()) {
+ case CASE:
+ caseOrdinal = i;
+ }
+ }
+ if (caseOrdinal < 0) {
+ return call;
+ }
+ // Convert
+ // f(CASE WHEN p1 THEN v1 ... END, arg)
+ // to
+ // CASE WHEN p1 THEN f(v1, arg) ... END
+ final RexCall case_ = (RexCall) operands.get(caseOrdinal);
+ final List<RexNode> nodes = new ArrayList<>();
+ for (int i = 0; i < case_.getOperands().size(); i++) {
+ RexNode node = case_.getOperands().get(i);
+ if (!RexUtil.isCasePredicate(case_, i)) {
+ node = substitute(call, caseOrdinal, node);
+ }
+ nodes.add(node);
+ }
+ return case_.clone(call.getType(), nodes);
+ }
+
+ /** Converts op(arg0, ..., argOrdinal, ..., argN) to op(arg0,..., node, ..., argN). */
+ protected static RexNode substitute(RexCall call, int ordinal, RexNode node) {
+ final List<RexNode> newOperands = Lists.newArrayList(call.getOperands());
+ newOperands.set(ordinal, node);
+ return call.clone(call.getType(), newOperands);
+ }
+
+ //~ Inner Classes ----------------------------------------------------------
+
+ /**
+ * Replaces expressions with their reductions. Note that we only have to
+ * look for RexCall, since nothing else is reducible in the first place.
+ */
+ protected static class RexReplacer extends RexShuttle {
+ private final RexBuilder rexBuilder;
+ private final List<RexNode> reducibleExps;
+ private final List<RexNode> reducedValues;
+ private final List<Boolean> addCasts;
+
+ RexReplacer(
+ RexBuilder rexBuilder,
+ List<RexNode> reducibleExps,
+ List<RexNode> reducedValues,
+ List<Boolean> addCasts) {
+ this.rexBuilder = rexBuilder;
+ this.reducibleExps = reducibleExps;
+ this.reducedValues = reducedValues;
+ this.addCasts = addCasts;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef inputRef) {
+ RexNode node = visit(inputRef);
+ if (node == null) {
+ return super.visitInputRef(inputRef);
+ }
+ return node;
+ }
+
+ @Override public RexNode visitCall(RexCall call) {
+ RexNode node = visit(call);
+ if (node != null) {
+ return node;
+ }
+ node = super.visitCall(call);
+ if (node != call) {
+ node = RexUtil.simplify(rexBuilder, node);
+ }
+ return node;
+ }
+
+ private RexNode visit(final RexNode call) {
+ int i = reducibleExps.indexOf(call);
+ if (i == -1) {
+ return null;
+ }
+ RexNode replacement = reducedValues.get(i);
+ if (addCasts.get(i)
+ && (replacement.getType() != call.getType())) {
+ // Handle change from nullable to NOT NULL by claiming
+ // that the result is still nullable, even though
+ // we know it isn't.
+ //
+ // Also, we cannot reduce CAST('abc' AS VARCHAR(4)) to 'abc'.
+ // If we make 'abc' of type VARCHAR(4), we may later encounter
+ // the same expression in a Project's digest where it has
+ // type VARCHAR(3), and that's wrong.
+ replacement = rexBuilder.makeAbstractCast(call.getType(), replacement);
+ }
+ return replacement;
+ }
+ }
+
+ /**
+ * Helper class used to locate expressions that either can be reduced to
+ * literals or contain redundant casts.
+ */
+ protected static class ReducibleExprLocator extends RexVisitorImpl<Void> {
+ /** Whether an expression is constant, and if so, whether it can be
+ * reduced to a simpler constant. */
+ enum Constancy {
+ NON_CONSTANT, REDUCIBLE_CONSTANT, IRREDUCIBLE_CONSTANT
+ }
+
+ private final RelDataTypeFactory typeFactory;
+
+ private final List<Constancy> stack;
+
+ private final ImmutableMap<RexNode, RexLiteral> constants;
+
+ private final List<RexNode> constExprs;
+
+ private final List<Boolean> addCasts;
+
+ private final List<RexNode> removableCasts;
+
+ private final List<SqlOperator> parentCallTypeStack;
+
+ ReducibleExprLocator(RelDataTypeFactory typeFactory,
+ ImmutableMap<RexNode, RexLiteral> constants, List<RexNode> constExprs,
+ List<Boolean> addCasts, List<RexNode> removableCasts) {
+ // go deep
+ super(true);
+ this.typeFactory = typeFactory;
+ this.constants = constants;
+ this.constExprs = constExprs;
+ this.addCasts = addCasts;
+ this.removableCasts = removableCasts;
+ this.stack = Lists.newArrayList();
+ this.parentCallTypeStack = Lists.newArrayList();
+ }
+
+ public void analyze(RexNode exp) {
+ assert stack.isEmpty();
+
+ exp.accept(this);
+
+ // Deal with top of stack
+ assert stack.size() == 1;
+ assert parentCallTypeStack.isEmpty();
+ Constancy rootConstancy = stack.get(0);
+ if (rootConstancy == Constancy.REDUCIBLE_CONSTANT) {
+ // The entire subtree was constant, so add it to the result.
+ addResult(exp);
+ }
+ stack.clear();
+ }
+
+ private Void pushVariable() {
+ stack.add(Constancy.NON_CONSTANT);
+ return null;
+ }
+
+ private void addResult(RexNode exp) {
+ // Cast of literal can't be reduced, so skip those (otherwise we'd
+ // go into an infinite loop as we add them back).
+ if (exp.getKind() == SqlKind.CAST) {
+ RexCall cast = (RexCall) exp;
+ RexNode operand = cast.getOperands().get(0);
+ if (operand instanceof RexLiteral) {
+ return;
+ }
+ }
+ constExprs.add(exp);
+
+ // In the case where the expression corresponds to a UDR argument,
+ // we need to preserve casts. Note that this only applies to
+ // the topmost argument, not expressions nested within the UDR
+ // call.
+ //
+ // REVIEW zfong 6/13/08 - Are there other expressions where we
+ // also need to preserve casts?
+ if (parentCallTypeStack.isEmpty()) {
+ addCasts.add(false);
+ } else {
+ addCasts.add(isUdf(Stacks.peek(parentCallTypeStack)));
+ }
+ }
+
+ private Boolean isUdf(SqlOperator operator) {
+ // return operator instanceof UserDefinedRoutine
+ return false;
+ }
+
+ public Void visitInputRef(RexInputRef inputRef) {
+ if (constants.containsKey(inputRef)) {
+ stack.add(Constancy.REDUCIBLE_CONSTANT);
+ return null;
+ }
+ return pushVariable();
+ }
+
+ public Void visitLiteral(RexLiteral literal) {
+ stack.add(Constancy.IRREDUCIBLE_CONSTANT);
+ return null;
+ }
+
+ public Void visitOver(RexOver over) {
+ // assume non-constant (running SUM(1) looks constant but isn't)
+ analyzeCall(over, Constancy.NON_CONSTANT);
+ return null;
+ }
+
+ public Void visitCorrelVariable(RexCorrelVariable correlVariable) {
+ return pushVariable();
+ }
+
+ public Void visitCall(RexCall call) {
+ // assume REDUCIBLE_CONSTANT until proven otherwise
+ analyzeCall(call, Constancy.REDUCIBLE_CONSTANT);
+ return null;
+ }
+
+ private void analyzeCall(RexCall call, Constancy callConstancy) {
+ Stacks.push(parentCallTypeStack, call.getOperator());
+
+ // visit operands, pushing their states onto stack
+ super.visitCall(call);
+
+ // look for NON_CONSTANT operands
+ int operandCount = call.getOperands().size();
+ List<Constancy> operandStack = Util.last(stack, operandCount);
+ for (Constancy operandConstancy : operandStack) {
+ if (operandConstancy == Constancy.NON_CONSTANT) {
+ callConstancy = Constancy.NON_CONSTANT;
+ }
+ }
+
+ // Even if all operands are constant, the call itself may
+ // be non-deterministic.
+ if (!call.getOperator().isDeterministic()) {
+ callConstancy = Constancy.NON_CONSTANT;
+ } else if (call.getOperator().isDynamicFunction()) {
+ // We can reduce the call to a constant, but we can't
+ // cache the plan if the function is dynamic.
+ // For now, treat it same as non-deterministic.
+ callConstancy = Constancy.NON_CONSTANT;
+ }
+
+ // Row operator itself can't be reduced to a literal, but if
+ // the operands are constants, we still want to reduce those
+ if ((callConstancy == Constancy.REDUCIBLE_CONSTANT)
+ && (call.getOperator() instanceof SqlRowOperator)) {
+ callConstancy = Constancy.NON_CONSTANT;
+ }
+
+ if (callConstancy == Constancy.NON_CONSTANT) {
+ // any REDUCIBLE_CONSTANT children are now known to be maximal
+ // reducible subtrees, so they can be added to the result
+ // list
+ for (int iOperand = 0; iOperand < operandCount; ++iOperand) {
+ Constancy constancy = operandStack.get(iOperand);
+ if (constancy == Constancy.REDUCIBLE_CONSTANT) {
+ addResult(call.getOperands().get(iOperand));
+ }
+ }
+
+ // if this cast expression can't be reduced to a literal,
+ // then see if we can remove the cast
+ if (call.getOperator() == SqlStdOperatorTable.CAST) {
+ reduceCasts(call);
+ }
+ }
+
+ // pop operands off of the stack
+ operandStack.clear();
+
+ // pop this parent call operator off the stack
+ Stacks.pop(parentCallTypeStack, call.getOperator());
+
+ // push constancy result for this call onto stack
+ stack.add(callConstancy);
+ }
+
+ private void reduceCasts(RexCall outerCast) {
+ List<RexNode> operands = outerCast.getOperands();
+ if (operands.size() != 1) {
+ return;
+ }
+ RelDataType outerCastType = outerCast.getType();
+ RelDataType operandType = operands.get(0).getType();
+ if (operandType.equals(outerCastType)) {
+ removableCasts.add(outerCast);
+ return;
+ }
+
+ // See if the reduction
+ // CAST((CAST x AS type) AS type NOT NULL)
+ // -> CAST(x AS type NOT NULL)
+ // applies. TODO jvs 15-Dec-2008: consider
+ // similar cases for precision changes.
+ if (!(operands.get(0) instanceof RexCall)) {
+ return;
+ }
+ RexCall innerCast = (RexCall) operands.get(0);
+ if (innerCast.getOperator() != SqlStdOperatorTable.CAST) {
+ return;
+ }
+ if (innerCast.getOperands().size() != 1) {
+ return;
+ }
+ RelDataType outerTypeNullable =
+ typeFactory.createTypeWithNullability(outerCastType, true);
+ RelDataType innerTypeNullable =
+ typeFactory.createTypeWithNullability(operandType, true);
+ if (outerTypeNullable != innerTypeNullable) {
+ return;
+ }
+ if (operandType.isNullable()) {
+ removableCasts.add(innerCast);
+ }
+ }
+
+ public Void visitDynamicParam(RexDynamicParam dynamicParam) {
+ return pushVariable();
+ }
+
+ public Void visitRangeRef(RexRangeRef rangeRef) {
+ return pushVariable();
+ }
+
+ public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+ return pushVariable();
+ }
+ }
+
+ /** Shuttle that pushes predicates into a CASE. */
+ protected static class CaseShuttle extends RexShuttle {
+ @Override public RexNode visitCall(RexCall call) {
+ for (;;) {
+ call = (RexCall) super.visitCall(call);
+ final RexCall old = call;
+ call = pushPredicateIntoCase(call);
+ if (call == old) {
+ return call;
+ }
+ }
+ }
+ }
+}
+
+// End HiveReduceExpressionsRule.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
index 425514d..d39744b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
@@ -150,11 +150,33 @@ class ASTBuilder {
switch (sqlType) {
case BINARY:
- ByteString bs = (ByteString) literal.getValue();
- val = bs.byteAt(0);
- type = HiveParser.BigintLiteral;
+ case DATE:
+ case TIME:
+ case TIMESTAMP:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_DAY_TIME:
+ if (literal.getValue() == null) {
+ return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node();
+ }
break;
case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ case DOUBLE:
+ case DECIMAL:
+ case FLOAT:
+ case REAL:
+ case VARCHAR:
+ case CHAR:
+ case BOOLEAN:
+ if (literal.getValue3() == null) {
+ return ASTBuilder.construct(HiveParser.TOK_NULL, "TOK_NULL").node();
+ }
+ }
+
+ switch (sqlType) {
+ case TINYINT:
if (useTypeQualInLiteral) {
val = literal.getValue3() + "Y";
} else {
@@ -244,6 +266,8 @@ class ASTBuilder {
type = HiveParser.TOK_NULL;
break;
+ //binary type should not be seen.
+ case BINARY:
default:
throw new RuntimeException("Unsupported Type: " + sqlType);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index 631a4ca..122546f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -46,12 +46,9 @@ import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ConversionUtil;
import org.apache.calcite.util.NlsString;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.type.Decimal128;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -69,8 +66,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
@@ -91,8 +88,6 @@ import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableMap;
public class RexNodeConverter {
- private static final Logger LOG = LoggerFactory.getLogger(RexNodeConverter.class);
-
private static class InputCtx {
private final RelDataType calciteInpDataType;
private final ImmutableMap<String, Integer> hiveNameToPosMap;
@@ -112,6 +107,11 @@ public class RexNodeConverter {
private final ImmutableList<InputCtx> inputCtxs;
private final boolean flattenExpr;
+ //Constructor used by HiveRexExecutorImpl
+ public RexNodeConverter(RelOptCluster cluster) {
+ this(cluster, new ArrayList<InputCtx>(), false);
+ }
+
public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
ImmutableMap<String, Integer> nameToPosMap, int offset, boolean flattenExpr) {
this.cluster = cluster;
@@ -259,6 +259,9 @@ public class RexNodeConverter {
GenericUDF udf = func.getGenericUDF();
if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar)
|| (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate)
+ // Calcite can not specify the scale for timestamp. As a result, all
+ // the millisecond part will be lost
+ || (udf instanceof GenericUDFTimestamp)
|| (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) {
castExpr = cluster.getRexBuilder().makeAbstractCast(
TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()),
@@ -321,6 +324,10 @@ public class RexNodeConverter {
coi);
RexNode calciteLiteral = null;
+ // If value is null, the type should also be VOID.
+ if (value == null) {
+ hiveTypeCategory = PrimitiveCategory.VOID;
+ }
// TODO: Verify if we need to use ConstantObjectInspector to unwrap data
switch (hiveTypeCategory) {
case BOOLEAN:
@@ -378,6 +385,10 @@ public class RexNodeConverter {
calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), calciteDataType);
break;
case DOUBLE:
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN((Double) value)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
calciteLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), calciteDataType);
break;
case CHAR:
@@ -417,14 +428,22 @@ public class RexNodeConverter {
new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1,1)));
break;
case INTERVAL_DAY_TIME:
+ // Calcite RexBuilder L525 divides value by the multiplier.
+ // Need to get CAlCITE-1020 in.
+ throw new CalciteSemanticException("INTERVAL_DAY_TIME is not well supported",
+ UnsupportedFeature.Invalid_interval);
// Calcite day-time interval is millis value as BigDecimal
// Seconds converted to millis
- BigDecimal secsValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
- // Nanos converted to millis
- BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime) value).getNanos(), 6);
- calciteLiteral = rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
- new SqlIntervalQualifier(TimeUnit.DAY, TimeUnit.SECOND, new SqlParserPos(1,1)));
- break;
+ // BigDecimal secsValueBd = BigDecimal
+ // .valueOf(((HiveIntervalDayTime) value).getTotalSeconds() * 1000);
+ // // Nanos converted to millis
+ // BigDecimal nanosValueBd = BigDecimal.valueOf(((HiveIntervalDayTime)
+ // value).getNanos(), 6);
+ // calciteLiteral =
+ // rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ // new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ // SqlParserPos(1, 1)));
+ // break;
case VOID:
calciteLiteral = cluster.getRexBuilder().makeLiteral(null,
cluster.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
@@ -438,11 +457,6 @@ public class RexNodeConverter {
return calciteLiteral;
}
- private RexNode createNullLiteral(ExprNodeDesc expr) throws CalciteSemanticException {
- return cluster.getRexBuilder().makeNullLiteral(
- TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName());
- }
-
public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode,
List<RelNode> inputRels, LinkedHashMap<RelNode, RowResolver> relToHiveRR,
Map<RelNode, ImmutableMap<String, Integer>> relToHiveColNameCalcitePosMap, boolean flattenExpr)
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 87b18b7..21423c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -40,6 +40,7 @@ import org.antlr.runtime.tree.TreeVisitor;
import org.antlr.runtime.tree.TreeVisitorAction;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptPlanner.Executor;
import org.apache.calcite.plan.RelOptQuery;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptSchema;
@@ -119,6 +120,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveHepPlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveVolcanoPlannerContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
@@ -154,6 +156,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePreFilteringRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectMergeRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveProjectSortTransposeRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveReduceExpressionsRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRelFieldTrimmer;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveRulesRegistry;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSortJoinReduceRule;
@@ -881,9 +884,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Create MD provider
HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf);
+ // Create executor
+ Executor executorProvider = new HiveRexExecutorImpl(cluster);
+
// 2. Apply pre-join order optimizations
calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan,
- mdProvider.getMetadataProvider());
+ mdProvider.getMetadataProvider(), executorProvider);
// 3. Apply join order optimizations: reordering MST algorithm
// If join optimizations failed because of missing stats, we continue with
@@ -935,7 +941,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// 4. Run other optimizations that do not need stats
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
HepMatchOrder.BOTTOM_UP,
ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE,
new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY),
@@ -979,7 +985,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// aggregation columns (HIVE-10627)
if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) {
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Window fixing rule");
}
@@ -988,7 +994,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
// 8.1. Merge join into multijoin operators (if possible)
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER,
HiveJoinProjectTransposeRule.LEFT_PROJECT_INCLUDE_OUTER,
HiveJoinProjectTransposeRule.RIGHT_PROJECT_INCLUDE_OUTER,
@@ -998,15 +1004,15 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
HiveRelFactories.HIVE_BUILDER.create(cluster, null));
calciteOptimizedPlan = fieldTrimmer.trim(calciteOptimizedPlan);
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE,
new ProjectMergeRule(false, HiveRelFactories.HIVE_PROJECT_FACTORY));
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null,
new HiveFilterProjectTSTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
HiveProject.class, HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class));
// 8.2. Introduce exchange operators below join/multijoin operators
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN,
HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree");
@@ -1031,9 +1037,11 @@ public class CalcitePlanner extends SemanticAnalyzer {
* original plan
* @param mdProvider
* meta data provider
+ * @param executorProvider
+ * executor
* @return
*/
- private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider) {
+ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProvider mdProvider, Executor executorProvider) {
// TODO: Decorelation of subquery should be done before attempting
// Partition Pruning; otherwise Expression evaluation may try to execute
// corelated sub query.
@@ -1048,7 +1056,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Its not clear, if this rewrite is always performant on MR, since extra map phase
// introduced for 2nd MR job may offset gains of this multi-stage aggregation.
// We need a cost model for MR to enable this on MR.
- basePlan = hepPlan(basePlan, true, mdProvider, HiveExpandDistinctAggregatesRule.INSTANCE);
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HiveExpandDistinctAggregatesRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Distinct aggregate rewrite");
}
@@ -1059,7 +1067,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Ex: select * from R1 join R2 where ((R1.x=R2.x) and R1.y<10) or
// ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, false, mdProvider, HepMatchOrder.ARBITRARY,
+ basePlan = hepPlan(basePlan, false, mdProvider, null, HepMatchOrder.ARBITRARY,
HivePreFilteringRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, factor out common filter elements and separating deterministic vs non-deterministic UDF");
@@ -1070,7 +1078,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// TODO: Add in ReduceExpressionrules (Constant folding) to below once
// HIVE-11927 is fixed.
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN,
HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class,
HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule(
@@ -1078,6 +1086,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, PPD for old join syntax");
+
// TODO: Transitive inference, constant prop & Predicate push down has to
// do multiple passes till no more inference is left
// Currently doing so would result in a spin. Just checking for if inferred
@@ -1085,7 +1094,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// could have been mutated by constant folding/prop
// 4. Transitive inference for join on clauses
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, new HiveJoinPushTransitivePredicatesRule(
+ basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveJoinPushTransitivePredicatesRule(
Join.class, HiveRelFactories.HIVE_FILTER_FACTORY));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Transitive inference for join on clauses");
@@ -1102,10 +1111,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_PERCENTAGE);
final long reductionTuples = HiveConf.getLongVar(conf,
HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE_REDUCTION_TUPLES);
- basePlan = hepPlan(basePlan, true, mdProvider, HiveSortMergeRule.INSTANCE,
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HiveSortMergeRule.INSTANCE,
HiveSortProjectTransposeRule.INSTANCE, HiveSortJoinReduceRule.INSTANCE,
HiveSortUnionReduceRule.INSTANCE);
- basePlan = hepPlan(basePlan, true, mdProvider, HepMatchOrder.BOTTOM_UP,
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HepMatchOrder.BOTTOM_UP,
new HiveSortRemoveRule(reductionProportion, reductionTuples),
HiveProjectSortTransposeRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
@@ -1114,7 +1123,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// 6. Add not null filters
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, HiveJoinAddNotNullRule.INSTANCE);
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HiveJoinAddNotNullRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Add not null filters");
@@ -1122,7 +1131,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// TODO: Add in ReduceExpressionrules (Constant folding) to below once
// HIVE-11927 is fixed.
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
+ basePlan = hepPlan(basePlan, true, mdProvider, null, HiveFilterProjectTransposeRule.INSTANCE_DETERMINISTIC,
HiveFilterSetOpTransposeRule.INSTANCE, HiveFilterSortTransposeRule.INSTANCE, HiveFilterJoinRule.JOIN,
HiveFilterJoinRule.FILTER_ON_JOIN, new HiveFilterAggregateTransposeRule(Filter.class,
HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class), new FilterMergeRule(
@@ -1132,18 +1141,26 @@ public class CalcitePlanner extends SemanticAnalyzer {
// 8. Push Down Semi Joins
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, SemiJoinJoinTransposeRule.INSTANCE,
+ basePlan = hepPlan(basePlan, true, mdProvider, null, SemiJoinJoinTransposeRule.INSTANCE,
SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Push Down Semi Joins");
- // 9. Apply Partition Pruning
+ // 9. Constant folding
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
+ basePlan = hepPlan(basePlan, true, mdProvider, executorProvider,
+ HiveReduceExpressionsRule.PROJECT_INSTANCE, HiveReduceExpressionsRule.FILTER_INSTANCE,
+ HiveReduceExpressionsRule.JOIN_INSTANCE);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
+ "Calcite: Prejoin ordering transformation, Constant folding");
+
+ // 10. Apply Partition Pruning
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, false, mdProvider, new HivePartitionPruneRule(conf));
+ basePlan = hepPlan(basePlan, false, mdProvider, null, new HivePartitionPruneRule(conf));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Partition Pruning");
- // 10. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP)
+ // 11. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP)
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null,
HiveRelFactories.HIVE_BUILDER.create(cluster, null));
@@ -1151,19 +1168,19 @@ public class CalcitePlanner extends SemanticAnalyzer {
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Projection Pruning");
- // 11. Merge Project-Project if possible
+ // 12. Merge Project-Project if possible
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, false, mdProvider, new ProjectMergeRule(true,
+ basePlan = hepPlan(basePlan, false, mdProvider, null, new ProjectMergeRule(true,
HiveRelFactories.HIVE_PROJECT_FACTORY));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
"Calcite: Prejoin ordering transformation, Merge Project-Project");
- // 12. Rerun PPD through Project as column pruning would have introduced
+ // 13. Rerun PPD through Project as column pruning would have introduced
// DT above scans; By pushing filter just above TS, Hive can push it into
// storage (incase there are filters on non partition cols). This only
// matches FIL-PROJ-TS
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
- basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTSTransposeRule(
+ basePlan = hepPlan(basePlan, true, mdProvider, null, new HiveFilterProjectTSTransposeRule(
Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, HiveProject.class,
HiveRelFactories.HIVE_PROJECT_FACTORY, HiveTableScan.class));
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER,
@@ -1178,12 +1195,13 @@ public class CalcitePlanner extends SemanticAnalyzer {
* @param basePlan
* @param followPlanChanges
* @param mdProvider
+ * @param executorProvider
* @param rules
* @return optimized RelNode
*/
private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges,
- RelMetadataProvider mdProvider, RelOptRule... rules) {
- return hepPlan(basePlan, followPlanChanges, mdProvider,
+ RelMetadataProvider mdProvider, Executor executorProvider, RelOptRule... rules) {
+ return hepPlan(basePlan, followPlanChanges, mdProvider, executorProvider,
HepMatchOrder.TOP_DOWN, rules);
}
@@ -1193,12 +1211,14 @@ public class CalcitePlanner extends SemanticAnalyzer {
* @param basePlan
* @param followPlanChanges
* @param mdProvider
+ * @param executorProvider
* @param order
* @param rules
* @return optimized RelNode
*/
- private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges, RelMetadataProvider mdProvider,
- HepMatchOrder order, RelOptRule... rules) {
+ private RelNode hepPlan(RelNode basePlan, boolean followPlanChanges,
+ RelMetadataProvider mdProvider, Executor executorProvider, HepMatchOrder order,
+ RelOptRule... rules) {
RelNode optimizedRelNode = basePlan;
HepProgramBuilder programBuilder = new HepProgramBuilder();
@@ -1214,6 +1234,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveRulesRegistry registry = new HiveRulesRegistry();
HiveHepPlannerContext context = new HiveHepPlannerContext(registry);
HepPlanner planner = new HepPlanner(programBuilder.build(), context);
+
List<RelMetadataProvider> list = Lists.newArrayList();
list.add(mdProvider);
planner.registerMetadataProviders(list);
@@ -1221,10 +1242,9 @@ public class CalcitePlanner extends SemanticAnalyzer {
basePlan.getCluster().setMetadataProvider(
new CachingRelMetadataProvider(chainedProvider, planner));
- // Executor is required for constant-reduction rules; see [CALCITE-566]
- final RexExecutorImpl executor =
- new RexExecutorImpl(Schemas.createDataContext(null));
- basePlan.getCluster().getPlanner().setExecutor(executor);
+ if (executorProvider != null) {
+ basePlan.getCluster().getPlanner().setExecutor(executorProvider);
+ }
planner.setRoot(basePlan);
optimizedRelNode = planner.findBestExp();
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/cbo_const.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/cbo_const.q b/ql/src/test/queries/clientpositive/cbo_const.q
new file mode 100644
index 0000000..e38e2da
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_const.q
@@ -0,0 +1,52 @@
+set hive.mapred.mode=nonstrict;
+set hive.cbo.enable=true;
+
+select
+ interval_day_time('2 1:2:3'),
+ interval_day_time(cast('2 1:2:3' as string)),
+ interval_day_time(cast('2 1:2:3' as varchar(10))),
+ interval_day_time(cast('2 1:2:3' as char(10))),
+ interval_day_time('2 1:2:3') = interval '2 1:2:3' day to second
+from src limit 1;
+
+select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08';
+
+drop view t1;
+
+create table t1_new (key string, value string) partitioned by (ds string);
+
+insert overwrite table t1_new partition (ds = '2011-10-15')
+select 'key1', 'value1' from src tablesample (1 rows);
+
+insert overwrite table t1_new partition (ds = '2011-10-16')
+select 'key2', 'value2' from src tablesample (1 rows);
+
+create view t1 partitioned on (ds) as
+select * from
+(
+select key, value, ds from t1_new
+union all
+select key, value, ds from t1_new
+)subq;
+
+select * from t1 where ds = '2011-10-15';
+
+
+explain select array(1,2,3) from src;
+
+EXPLAIN
+select key from (SELECT key from src where key = 1+3)s;
+
+select * from (select key from src where key = '1')subq;
+
+select '1';
+
+select * from (select '1')subq;
+
+select * from (select key from src where false)subq;
+
+EXPLAIN
+SELECT x.key, z.value, y.value
+FROM src1 x JOIN src y ON (x.key = y.key and y.key = 1+2)
+JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11+3);
+
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/queries/clientpositive/constantfolding.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constantfolding.q b/ql/src/test/queries/clientpositive/constantfolding.q
new file mode 100644
index 0000000..4ddb710
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constantfolding.q
@@ -0,0 +1,88 @@
+set hive.mapred.mode=nonstrict;
+set hive.optimize.ppd=true;
+
+-- SORT_QUERY_RESULTS
+
+select * from (select 'k2' as key, '1 ' as value from src limit 2)b
+union all
+select * from (select 'k3' as key, '' as value from src limit 2)b
+union all
+select * from (select 'k4' as key, ' ' as value from src limit 2)c;
+
+
+drop table if exists union_all_bug_test_1;
+drop table if exists union_all_bug_test_2;
+create table if not exists union_all_bug_test_1
+(
+f1 int,
+f2 int
+);
+
+create table if not exists union_all_bug_test_2
+(
+f1 int
+);
+
+insert into table union_all_bug_test_1 values (1,1);
+insert into table union_all_bug_test_2 values (1);
+insert into table union_all_bug_test_1 values (0,0);
+insert into table union_all_bug_test_2 values (0);
+
+
+
+SELECT f1
+FROM (
+
+SELECT
+f1
+, if('helloworld' like '%hello%' ,f1,f2) as filter
+FROM union_all_bug_test_1
+
+union all
+
+select
+f1
+, 0 as filter
+from union_all_bug_test_2
+) A
+WHERE (filter = 1 and f1 = 1);
+
+
+select percentile(cast(key as bigint), array()) from src where false;
+
+select unbase64("0xe23") from src limit 1;
+
+SELECT key,randum123, h4
+FROM (SELECT *, cast(rand() as double) AS randum123, hex(4) AS h4 FROM src WHERE key = 100) a
+WHERE a.h4 <= 3 limit 1;
+
+select null from src limit 1;
+
+-- numRows: 2 rawDataSize: 80
+explain select cast("1970-12-31 15:59:58.174" as TIMESTAMP) from src;
+
+-- numRows: 2 rawDataSize: 112
+explain select cast("1970-12-31 15:59:58.174" as DATE) from src;
+
+CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE;
+
+FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86;
+
+EXPLAIN
+SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
+ LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1),
+ ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12),
+ LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12),
+ POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+ POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)),
+ POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)),
+ POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1;
+
+SELECT ROUND(LN(3.0),12), LN(0.0), LN(-1), ROUND(LOG(3.0),12), LOG(0.0),
+ LOG(-1), ROUND(LOG2(3.0),12), LOG2(0.0), LOG2(-1),
+ ROUND(LOG10(3.0),12), LOG10(0.0), LOG10(-1), ROUND(LOG(2, 3.0),12),
+ LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), ROUND(EXP(2.0),12),
+ POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+ POWER(-1, 0.5), POWER(-1, 2), POWER(CAST (1 AS DECIMAL), CAST (0 AS INT)),
+ POWER(CAST (2 AS DECIMAL), CAST (3 AS INT)),
+ POW(CAST (2 AS DECIMAL), CAST(3 AS INT)) FROM dest1;
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/annotate_stats_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_select.q.out b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
index c4d59c8..b158d85 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_select.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_select.q.out
@@ -925,24 +925,46 @@ POSTHOOK: query: -- inner select - numRows: 2 rawDataSize: 24
explain select x from (select i1,11.0 as x from alltypes_orc limit 10) temp
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-0 is a root stage
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-0
- Fetch Operator
- limit: 10
- Processor Tree:
- TableScan
- alias: alltypes_orc
- Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: alltypes_orc
+ Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
+ Reduce Operator Tree:
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: 11.0 (type: double)
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 10
+ File Output Operator
+ compressed: false
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- ListSink
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
PREHOOK: query: -- inner select - numRows: 2 rawDataSize: 104
-- outer select - numRows: 2 rawDataSize: 186
@@ -1024,21 +1046,21 @@ STAGE PLANS:
alias: alltypes_orc
Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Reduce Operator Tree:
Limit
Number of rows: 10
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -1052,12 +1074,12 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
Reduce Operator Tree:
Limit
Number of rows: 10
- Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: 'hello' (type: string), 11.0 (type: double)
outputColumnNames: _col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
index 277b0f7..cfb95be 100644
--- a/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
+++ b/ql/src/test/results/clientpositive/bucketizedhiveinputformat.q.out
@@ -22,6 +22,8 @@ POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@T2
+Warning: Shuffle Join JOIN[13][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (
SELECT tmp1.name as name FROM (
SELECT name, 'MMM' AS n FROM T1) tmp1
http://git-wip-us.apache.org/repos/asf/hive/blob/b340ecb5/ql/src/test/results/clientpositive/cast1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cast1.q.out b/ql/src/test/results/clientpositive/cast1.q.out
index 0bdecba..48a0c14 100644
--- a/ql/src/test/results/clientpositive/cast1.q.out
+++ b/ql/src/test/results/clientpositive/cast1.q.out
@@ -105,11 +105,11 @@ POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
-POSTHOOK: Lineage: dest1.c1 EXPRESSION []
-POSTHOOK: Lineage: dest1.c2 EXPRESSION []
-POSTHOOK: Lineage: dest1.c3 EXPRESSION []
-POSTHOOK: Lineage: dest1.c4 EXPRESSION []
-POSTHOOK: Lineage: dest1.c5 EXPRESSION []
+POSTHOOK: Lineage: dest1.c1 SIMPLE []
+POSTHOOK: Lineage: dest1.c2 SIMPLE []
+POSTHOOK: Lineage: dest1.c3 SIMPLE []
+POSTHOOK: Lineage: dest1.c4 SIMPLE []
+POSTHOOK: Lineage: dest1.c5 SIMPLE []
POSTHOOK: Lineage: dest1.c6 EXPRESSION []
POSTHOOK: Lineage: dest1.c7 EXPRESSION []
PREHOOK: query: select dest1.* FROM dest1