You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/10/30 17:22:48 UTC
svn commit: r1635536 [11/28] - in /hive/branches/spark: ./ accumulo-handler/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/
accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/
accumulo-handler/src/test/org/apache/hado...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java Thu Oct 30 16:22:33 2014
@@ -77,7 +77,8 @@ public class HiveOptiqUtil {
public static boolean validateASTForCBO(ASTNode ast) {
String astTree = ast.toStringTree();
- String[] tokens = { "TOK_CHARSETLITERAL" };
+ // if any of following tokens are present in AST, bail out
+ String[] tokens = { "TOK_CHARSETLITERAL","TOK_TABLESPLITSAMPLE" };
for (String token : tokens) {
if (astTree.contains(token)) {
return false;
@@ -241,7 +242,7 @@ public class HiveOptiqUtil {
* <p>
* JoinPredicateInfo:<br>
* 1. preserves the order of conjuctive elements for
- * equi-join(m_equiJoinPredicateElements)<br>
+ * equi-join(equiJoinPredicateElements)<br>
* 2. Stores set of projection indexes from left and right child which is part
* of equi join keys; the indexes are both in child and Join node schema.<br>
* 3. Keeps a map of projection indexes that are part of join keys to list of
@@ -275,19 +276,19 @@ public class HiveOptiqUtil {
}
public List<JoinLeafPredicateInfo> getNonEquiJoinPredicateElements() {
- return nonEquiJoinPredicateElements;
+ return this.nonEquiJoinPredicateElements;
}
public List<JoinLeafPredicateInfo> getEquiJoinPredicateElements() {
- return equiJoinPredicateElements;
+ return this.equiJoinPredicateElements;
}
public Set<Integer> getProjsFromLeftPartOfJoinKeysInChildSchema() {
- return projsFromLeftPartOfJoinKeysInChildSchema;
+ return this.projsFromLeftPartOfJoinKeysInChildSchema;
}
public Set<Integer> getProjsFromRightPartOfJoinKeysInChildSchema() {
- return projsFromRightPartOfJoinKeysInChildSchema;
+ return this.projsFromRightPartOfJoinKeysInChildSchema;
}
/**
@@ -296,15 +297,15 @@ public class HiveOptiqUtil {
* schema.
*/
public Set<Integer> getProjsFromLeftPartOfJoinKeysInJoinSchema() {
- return projsFromLeftPartOfJoinKeysInChildSchema;
+ return this.projsFromLeftPartOfJoinKeysInChildSchema;
}
public Set<Integer> getProjsFromRightPartOfJoinKeysInJoinSchema() {
- return projsFromRightPartOfJoinKeysInJoinSchema;
+ return this.projsFromRightPartOfJoinKeysInJoinSchema;
}
public Map<Integer, ImmutableList<JoinLeafPredicateInfo>> getMapOfProjIndxToLeafPInfo() {
- return mapOfProjIndxInJoinSchemaToLeafPInfo;
+ return this.mapOfProjIndxInJoinSchemaToLeafPInfo;
}
public static JoinPredicateInfo constructJoinPredicateInfo(HiveJoinRel j) {
@@ -336,7 +337,7 @@ public class HiveOptiqUtil {
jlpi = JoinLeafPredicateInfo.constructJoinLeafPredicateInfo(j, ce);
// 2.2 Classify leaf predicate as Equi vs Non Equi
- if (jlpi.m_comparisonType.equals(SqlKind.EQUALS)) {
+ if (jlpi.comparisonType.equals(SqlKind.EQUALS)) {
equiLPIList.add(jlpi);
} else {
nonEquiLPIList.add(jlpi);
@@ -398,38 +399,38 @@ public class HiveOptiqUtil {
* of equi join keys; the indexes are both in child and Join node schema.<br>
*/
public static class JoinLeafPredicateInfo {
- private final SqlKind m_comparisonType;
- private final ImmutableList<RexNode> m_joinKeyExprsFromLeft;
- private final ImmutableList<RexNode> m_joinKeyExprsFromRight;
- private final ImmutableSet<Integer> m_projsFromLeftPartOfJoinKeysInChildSchema;
- private final ImmutableSet<Integer> m_projsFromRightPartOfJoinKeysInChildSchema;
- private final ImmutableSet<Integer> m_projsFromRightPartOfJoinKeysInJoinSchema;
+ private final SqlKind comparisonType;
+ private final ImmutableList<RexNode> joinKeyExprsFromLeft;
+ private final ImmutableList<RexNode> joinKeyExprsFromRight;
+ private final ImmutableSet<Integer> projsFromLeftPartOfJoinKeysInChildSchema;
+ private final ImmutableSet<Integer> projsFromRightPartOfJoinKeysInChildSchema;
+ private final ImmutableSet<Integer> projsFromRightPartOfJoinKeysInJoinSchema;
public JoinLeafPredicateInfo(SqlKind comparisonType, List<RexNode> joinKeyExprsFromLeft,
List<RexNode> joinKeyExprsFromRight, Set<Integer> projsFromLeftPartOfJoinKeysInChildSchema,
Set<Integer> projsFromRightPartOfJoinKeysInChildSchema,
Set<Integer> projsFromRightPartOfJoinKeysInJoinSchema) {
- m_comparisonType = comparisonType;
- m_joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft);
- m_joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight);
- m_projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet
+ this.comparisonType = comparisonType;
+ this.joinKeyExprsFromLeft = ImmutableList.copyOf(joinKeyExprsFromLeft);
+ this.joinKeyExprsFromRight = ImmutableList.copyOf(joinKeyExprsFromRight);
+ this.projsFromLeftPartOfJoinKeysInChildSchema = ImmutableSet
.copyOf(projsFromLeftPartOfJoinKeysInChildSchema);
- m_projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet
+ this.projsFromRightPartOfJoinKeysInChildSchema = ImmutableSet
.copyOf(projsFromRightPartOfJoinKeysInChildSchema);
- m_projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet
+ this.projsFromRightPartOfJoinKeysInJoinSchema = ImmutableSet
.copyOf(projsFromRightPartOfJoinKeysInJoinSchema);
}
public List<RexNode> getJoinKeyExprsFromLeft() {
- return m_joinKeyExprsFromLeft;
+ return this.joinKeyExprsFromLeft;
}
public List<RexNode> getJoinKeyExprsFromRight() {
- return m_joinKeyExprsFromRight;
+ return this.joinKeyExprsFromRight;
}
public Set<Integer> getProjsFromLeftPartOfJoinKeysInChildSchema() {
- return m_projsFromLeftPartOfJoinKeysInChildSchema;
+ return this.projsFromLeftPartOfJoinKeysInChildSchema;
}
/**
@@ -438,15 +439,15 @@ public class HiveOptiqUtil {
* schema.
*/
public Set<Integer> getProjsFromLeftPartOfJoinKeysInJoinSchema() {
- return m_projsFromLeftPartOfJoinKeysInChildSchema;
+ return this.projsFromLeftPartOfJoinKeysInChildSchema;
}
public Set<Integer> getProjsFromRightPartOfJoinKeysInChildSchema() {
- return m_projsFromRightPartOfJoinKeysInChildSchema;
+ return this.projsFromRightPartOfJoinKeysInChildSchema;
}
public Set<Integer> getProjsFromRightPartOfJoinKeysInJoinSchema() {
- return m_projsFromRightPartOfJoinKeysInJoinSchema;
+ return this.projsFromRightPartOfJoinKeysInJoinSchema;
}
private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(HiveJoinRel j, RexNode pe) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java Thu Oct 30 16:22:33 2014
@@ -17,274 +17,135 @@
*/
package org.apache.hadoop.hive.ql.optimizer.optiq.rules;
-import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.ListIterator;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel;
-import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel;
-import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveRel;
+import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.JoinRelBase;
import org.eigenbase.rel.JoinRelType;
-import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.RelFactories;
+import org.eigenbase.rel.rules.PushFilterPastJoinRule;
import org.eigenbase.relopt.RelOptRule;
import org.eigenbase.relopt.RelOptRuleCall;
import org.eigenbase.relopt.RelOptRuleOperand;
-import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.relopt.RelOptUtil.InputFinder;
-import org.eigenbase.rex.RexBuilder;
import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexUtil;
import org.eigenbase.sql.SqlKind;
-import org.eigenbase.util.Holder;
-public abstract class HivePushFilterPastJoinRule extends RelOptRule {
+public abstract class HivePushFilterPastJoinRule extends PushFilterPastJoinRule {
- public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterPastJoinRule(
- operand(FilterRelBase.class, operand(HiveJoinRel.class, any())),
- "HivePushFilterPastJoinRule:filter", true) {
- @Override
- public void onMatch(RelOptRuleCall call) {
- HiveFilterRel filter = call.rel(0);
- HiveJoinRel join = call.rel(1);
- perform(call, filter, join);
- }
- };
-
- public static final HivePushFilterPastJoinRule JOIN = new HivePushFilterPastJoinRule(
- operand(HiveJoinRel.class, any()), "HivePushFilterPastJoinRule:no-filter", false) {
- @Override
- public void onMatch(RelOptRuleCall call) {
- HiveJoinRel join = call.rel(0);
- perform(call, null, join);
- }
- };
-
- /** Whether to try to strengthen join-type. */
- private final boolean smart;
-
- // ~ Constructors -----------------------------------------------------------
-
- /**
- * Creates a PushFilterPastJoinRule with an explicit root operand.
- */
- private HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id, boolean smart) {
- super(operand, "PushFilterRule: " + id);
- this.smart = smart;
- }
-
- // ~ Methods ----------------------------------------------------------------
-
- protected void perform(RelOptRuleCall call, FilterRelBase filter,
- JoinRelBase join) {
- final List<RexNode> joinFilters = RelOptUtil.conjunctions(join
- .getCondition());
-
- /*
- * todo: hb 6/26/14 for left SemiJoin we cannot push predicates yet. The
- * assertion that num(JoinRel columns) = num(leftSrc) + num(rightSrc)
- * doesn't hold. So RelOptUtil.classifyFilters fails.
- */
- if (((HiveJoinRel) join).isLeftSemiJoin()) {
- return;
- }
-
- if (filter == null) {
- // There is only the joinRel
- // make sure it does not match a cartesian product joinRel
- // (with "true" condition) otherwise this rule will be applied
- // again on the new cartesian product joinRel.
- boolean onlyTrueFilter = true;
- for (RexNode joinFilter : joinFilters) {
- if (!joinFilter.isAlwaysTrue()) {
- onlyTrueFilter = false;
- break;
- }
- }
-
- if (onlyTrueFilter) {
- return;
- }
- }
-
- final List<RexNode> aboveFilters = filter != null ? RelOptUtil
- .conjunctions(filter.getCondition()) : new ArrayList<RexNode>();
-
- List<RexNode> leftFilters = new ArrayList<RexNode>();
- List<RexNode> rightFilters = new ArrayList<RexNode>();
- int origJoinFiltersSz = joinFilters.size();
-
- // TODO - add logic to derive additional filters. E.g., from
- // (t1.a = 1 AND t2.a = 2) OR (t1.b = 3 AND t2.b = 4), you can
- // derive table filters:
- // (t1.a = 1 OR t1.b = 3)
- // (t2.a = 2 OR t2.b = 4)
-
- // Try to push down above filters. These are typically where clause
- // filters. They can be pushed down if they are not on the NULL
- // generating side.
- boolean filterPushed = false;
- final Holder<JoinRelType> joinTypeHolder = Holder.of(join.getJoinType());
- if (RelOptUtil.classifyFilters(join, aboveFilters,
- join.getJoinType(), true, !join.getJoinType().generatesNullsOnLeft(), !join.getJoinType()
- .generatesNullsOnRight(), joinFilters, leftFilters, rightFilters, joinTypeHolder, smart)) {
- filterPushed = true;
- }
-
- /*
- * Any predicates pushed down to joinFilters that aren't equality
- * conditions: put them back as aboveFilters because Hive doesn't support
- * not equi join conditions.
- */
- ListIterator<RexNode> filterIter = joinFilters.listIterator();
- while (filterIter.hasNext()) {
- RexNode exp = filterIter.next();
- if (exp instanceof RexCall) {
- RexCall c = (RexCall) exp;
- if (c.getOperator().getKind() == SqlKind.EQUALS) {
- boolean validHiveJoinFilter = true;
- for (RexNode rn : c.getOperands()) {
- // NOTE: Hive dis-allows projections from both left & right side
- // of join condition. Example: Hive disallows
- // (r1.x=r2.x)=(r1.y=r2.y) on join condition.
- if (filterRefersToBothSidesOfJoin(rn, join)) {
- validHiveJoinFilter = false;
- break;
- }
- }
- if (validHiveJoinFilter)
- continue;
- }
- }
- aboveFilters.add(exp);
- filterIter.remove();
- }
-
- /*
- * if all pushed filters where put back then set filterPushed to false
- */
- if (leftFilters.size() == 0 && rightFilters.size() == 0
- && joinFilters.size() == origJoinFiltersSz) {
- filterPushed = false;
- }
-
- // Try to push down filters in ON clause. A ON clause filter can only be
- // pushed down if it does not affect the non-matching set, i.e. it is
- // not on the side which is preserved.
- if (RelOptUtil.classifyFilters(join, joinFilters, null, false, !join
- .getJoinType().generatesNullsOnRight(), !join.getJoinType()
- .generatesNullsOnLeft(), joinFilters, leftFilters, rightFilters, joinTypeHolder, false)) {
- filterPushed = true;
- }
-
- if (!filterPushed) {
- return;
- }
-
- /*
- * Remove always true conditions that got pushed down.
- */
- removeAlwaysTruePredicates(leftFilters);
- removeAlwaysTruePredicates(rightFilters);
- removeAlwaysTruePredicates(joinFilters);
-
- // create FilterRels on top of the children if any filters were
- // pushed to them
- RexBuilder rexBuilder = join.getCluster().getRexBuilder();
- RelNode leftRel = createFilterOnRel(rexBuilder, join.getLeft(), leftFilters);
- RelNode rightRel = createFilterOnRel(rexBuilder, join.getRight(),
- rightFilters);
-
- // create the new join node referencing the new children and
- // containing its new join filters (if there are any)
- RexNode joinFilter;
-
- if (joinFilters.size() == 0) {
- // if nothing actually got pushed and there is nothing leftover,
- // then this rule is a no-op
- if (leftFilters.isEmpty()
- && rightFilters.isEmpty()
- && joinTypeHolder.get() == join.getJoinType()) {
- return;
- }
- joinFilter = rexBuilder.makeLiteral(true);
- } else {
- joinFilter = RexUtil.composeConjunction(rexBuilder, joinFilters, true);
- }
- RelNode newJoinRel = HiveJoinRel.getJoin(join.getCluster(), leftRel,
- rightRel, joinFilter, join.getJoinType(), false);
-
- // create a FilterRel on top of the join if needed
- RelNode newRel = createFilterOnRel(rexBuilder, newJoinRel, aboveFilters);
-
- call.transformTo(newRel);
- }
-
- /**
- * If the filter list passed in is non-empty, creates a FilterRel on top of
- * the existing RelNode; otherwise, just returns the RelNode
- *
- * @param rexBuilder
- * rex builder
- * @param rel
- * the RelNode that the filter will be put on top of
- * @param filters
- * list of filters
- * @return new RelNode or existing one if no filters
- */
- private RelNode createFilterOnRel(RexBuilder rexBuilder, RelNode rel,
- List<RexNode> filters) {
- RexNode andFilters = RexUtil.composeConjunction(rexBuilder, filters, false);
- if (andFilters.isAlwaysTrue()) {
- return rel;
- }
- return new HiveFilterRel(rel.getCluster(), rel.getCluster().traitSetOf(
- HiveRel.CONVENTION), rel, andFilters);
- }
-
- private void removeAlwaysTruePredicates(List<RexNode> predicates) {
-
- ListIterator<RexNode> iter = predicates.listIterator();
- while (iter.hasNext()) {
- RexNode exp = iter.next();
- if (isAlwaysTrue(exp)) {
- iter.remove();
- }
- }
- }
-
- private boolean isAlwaysTrue(RexNode predicate) {
- if (predicate instanceof RexCall) {
- RexCall c = (RexCall) predicate;
- if (c.getOperator().getKind() == SqlKind.EQUALS) {
- return isAlwaysTrue(c.getOperands().get(0))
- && isAlwaysTrue(c.getOperands().get(1));
- }
- }
- return predicate.isAlwaysTrue();
- }
-
- private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) {
- boolean refersToBothSides = false;
-
- int joinNoOfProjects = j.getRowType().getFieldCount();
- BitSet filterProjs = new BitSet(joinNoOfProjects);
- BitSet allLeftProjs = new BitSet(joinNoOfProjects);
- BitSet allRightProjs = new BitSet(joinNoOfProjects);
- allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true);
- allRightProjs.set(j.getInput(0).getRowType().getFieldCount(), joinNoOfProjects, true);
+ public static final HivePushFilterPastJoinRule FILTER_ON_JOIN = new HivePushFilterIntoJoinRule();
- InputFinder inputFinder = new InputFinder(filterProjs);
- filter.accept(inputFinder);
+ public static final HivePushFilterPastJoinRule JOIN = new HivePushDownJoinConditionRule();
- if (allLeftProjs.intersects(filterProjs) && allRightProjs.intersects(filterProjs))
- refersToBothSides = true;
+ /**
+ * Creates a PushFilterPastJoinRule with an explicit root operand.
+ */
+ protected HivePushFilterPastJoinRule(RelOptRuleOperand operand, String id,
+ boolean smart, RelFactories.FilterFactory filterFactory,
+ RelFactories.ProjectFactory projectFactory) {
+ super(operand, id, smart, filterFactory, projectFactory);
+ }
+
+ /**
+ * Rule that tries to push filter expressions into a join condition and into
+ * the inputs of the join.
+ */
+ public static class HivePushFilterIntoJoinRule extends
+ HivePushFilterPastJoinRule {
+ public HivePushFilterIntoJoinRule() {
+ super(RelOptRule.operand(FilterRelBase.class,
+ RelOptRule.operand(JoinRelBase.class, RelOptRule.any())),
+ "HivePushFilterPastJoinRule:filter", true,
+ HiveFilterRel.DEFAULT_FILTER_FACTORY,
+ HiveProjectRel.DEFAULT_PROJECT_FACTORY);
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ FilterRelBase filter = call.rel(0);
+ JoinRelBase join = call.rel(1);
+ super.perform(call, filter, join);
+ }
+ }
+
+ public static class HivePushDownJoinConditionRule extends
+ HivePushFilterPastJoinRule {
+ public HivePushDownJoinConditionRule() {
+ super(RelOptRule.operand(JoinRelBase.class, RelOptRule.any()),
+ "HivePushFilterPastJoinRule:no-filter", true,
+ HiveFilterRel.DEFAULT_FILTER_FACTORY,
+ HiveProjectRel.DEFAULT_PROJECT_FACTORY);
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ JoinRelBase join = call.rel(0);
+ super.perform(call, null, join);
+ }
+ }
+
+ /*
+ * Any predicates pushed down to joinFilters that aren't equality
+ * conditions: put them back as aboveFilters because Hive doesn't support
+ * not equi join conditions.
+ */
+ @Override
+ protected void validateJoinFilters(List<RexNode> aboveFilters,
+ List<RexNode> joinFilters, JoinRelBase join, JoinRelType joinType) {
+ if (joinType.equals(JoinRelType.INNER)) {
+ ListIterator<RexNode> filterIter = joinFilters.listIterator();
+ while (filterIter.hasNext()) {
+ RexNode exp = filterIter.next();
+ if (exp instanceof RexCall) {
+ RexCall c = (RexCall) exp;
+ if (c.getOperator().getKind() == SqlKind.EQUALS) {
+ boolean validHiveJoinFilter = true;
+ for (RexNode rn : c.getOperands()) {
+ // NOTE: Hive dis-allows projections from both left
+ // &
+ // right side
+ // of join condition. Example: Hive disallows
+ // (r1.x=r2.x)=(r1.y=r2.y) on join condition.
+ if (filterRefersToBothSidesOfJoin(rn, join)) {
+ validHiveJoinFilter = false;
+ break;
+ }
+ }
+ if (validHiveJoinFilter)
+ continue;
+ }
+ }
+ aboveFilters.add(exp);
+ filterIter.remove();
+ }
+ }
+ }
+
+ private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) {
+ boolean refersToBothSides = false;
+
+ int joinNoOfProjects = j.getRowType().getFieldCount();
+ BitSet filterProjs = new BitSet(joinNoOfProjects);
+ BitSet allLeftProjs = new BitSet(joinNoOfProjects);
+ BitSet allRightProjs = new BitSet(joinNoOfProjects);
+ allLeftProjs.set(0, j.getInput(0).getRowType().getFieldCount(), true);
+ allRightProjs.set(j.getInput(0).getRowType().getFieldCount(),
+ joinNoOfProjects, true);
+
+ InputFinder inputFinder = new InputFinder(filterProjs);
+ filter.accept(inputFinder);
+
+ if (allLeftProjs.intersects(filterProjs)
+ && allRightProjs.intersects(filterProjs))
+ refersToBothSides = true;
- return refersToBothSides;
- }
+ return refersToBothSides;
+ }
}
// End PushFilterPastJoinRule.java
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java Thu Oct 30 16:22:33 2014
@@ -34,13 +34,13 @@ import org.eigenbase.rex.RexVisitorImpl;
import org.eigenbase.sql.SqlKind;
public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
- private final RelNode m_childRel;
- private final double m_childCardinality;
+ private final RelNode childRel;
+ private final double childCardinality;
protected FilterSelectivityEstimator(RelNode childRel) {
super(true);
- m_childRel = childRel;
- m_childCardinality = RelMetadataQuery.getRowCount(m_childRel);
+ this.childRel = childRel;
+ this.childCardinality = RelMetadataQuery.getRowCount(childRel);
}
public Double estimateSelectivity(RexNode predicate) {
@@ -53,11 +53,10 @@ public class FilterSelectivityEstimator
}
/*
- * Ignore any predicates on partition columns
- * because we have already accounted for these in
- * the Table row count.
+ * Ignore any predicates on partition columns because we have already
+ * accounted for these in the Table row count.
*/
- if (isPartitionPredicate(call, m_childRel)) {
+ if (isPartitionPredicate(call, this.childRel)) {
return 1.0;
}
@@ -77,6 +76,7 @@ public class FilterSelectivityEstimator
case NOT_EQUALS: {
selectivity = computeNotEqualitySelectivity(call);
+ break;
}
case LESS_THAN_OR_EQUAL:
@@ -150,10 +150,10 @@ public class FilterSelectivityEstimator
if (tmpSelectivity == null) {
tmpSelectivity = 0.99;
}
- tmpCardinality = m_childCardinality * tmpSelectivity;
+ tmpCardinality = childCardinality * tmpSelectivity;
if (tmpCardinality > 1)
- tmpSelectivity = (1 - tmpCardinality / m_childCardinality);
+ tmpSelectivity = (1 - tmpCardinality / childCardinality);
else
tmpSelectivity = 1.0;
@@ -194,7 +194,7 @@ public class FilterSelectivityEstimator
for (RexNode op : call.getOperands()) {
if (op instanceof RexInputRef) {
- tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel,
+ tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel,
((RexInputRef) op).getIndex());
if (tmpNDV > maxNDV)
maxNDV = tmpNDV;
@@ -202,7 +202,7 @@ public class FilterSelectivityEstimator
irv = new InputReferencedVisitor();
irv.apply(op);
for (Integer childProjIndx : irv.inputPosReferenced) {
- tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(m_childRel, childProjIndx);
+ tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, childProjIndx);
if (tmpNDV > maxNDV)
maxNDV = tmpNDV;
}
@@ -213,14 +213,13 @@ public class FilterSelectivityEstimator
}
private boolean isPartitionPredicate(RexNode expr, RelNode r) {
- if ( r instanceof ProjectRelBase ) {
+ if (r instanceof ProjectRelBase) {
expr = RelOptUtil.pushFilterPastProject(expr, (ProjectRelBase) r);
return isPartitionPredicate(expr, ((ProjectRelBase) r).getChild());
- } else if ( r instanceof FilterRelBase ) {
+ } else if (r instanceof FilterRelBase) {
return isPartitionPredicate(expr, ((FilterRelBase) r).getChild());
- } else if ( r instanceof HiveTableScanRel ) {
- RelOptHiveTable table = (RelOptHiveTable)
- ((HiveTableScanRel)r).getTable();
+ } else if (r instanceof HiveTableScanRel) {
+ RelOptHiveTable table = (RelOptHiveTable) ((HiveTableScanRel) r).getTable();
BitSet cols = RelOptUtil.InputFinder.bits(expr);
return table.containsPartitionColumnsOnly(cols);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdRowCount.java Thu Oct 30 16:22:33 2014
@@ -28,6 +28,7 @@ import net.hydromatic.optiq.util.BitSets
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.JoinRelBase;
import org.eigenbase.rel.JoinRelType;
@@ -41,6 +42,7 @@ import org.eigenbase.rel.metadata.RelMet
import org.eigenbase.rel.metadata.RelMetadataQuery;
import org.eigenbase.rel.rules.SemiJoinRel;
import org.eigenbase.relopt.RelOptUtil;
+import org.eigenbase.relopt.hep.HepRelVertex;
import org.eigenbase.rex.RexBuilder;
import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexInputRef;
@@ -190,7 +192,7 @@ public class HiveRelMdRowCount extends R
RelOptUtil.classifyFilters(joinRel, joinFilters, joinRel.getJoinType(),
false, !joinRel.getJoinType().generatesNullsOnRight(), !joinRel
.getJoinType().generatesNullsOnLeft(), joinFilters, leftFilters,
- rightFilters, joinTypeHolder, false);
+ rightFilters);
Pair<Integer, Integer> joinCols = canHandleJoin(joinRel, leftFilters,
rightFilters, joinFilters);
@@ -270,10 +272,11 @@ public class HiveRelMdRowCount extends R
if (pkSide == 0) {
FKSideInfo fkInfo = new FKSideInfo(rightRowCount,
rightNDV);
+ double pkSelectivity = pkSelectivity(joinRel, true, left, leftRowCount);
PKSideInfo pkInfo = new PKSideInfo(leftRowCount,
leftNDV,
joinRel.getJoinType().generatesNullsOnRight() ? 1.0 :
- isPKSideSimpleTree ? RelMetadataQuery.getSelectivity(left, leftPred) : 1.0);
+ pkSelectivity);
return new PKFKRelationInfo(1, fkInfo, pkInfo, ndvScalingFactor, isPKSideSimpleTree);
}
@@ -281,10 +284,11 @@ public class HiveRelMdRowCount extends R
if (pkSide == 1) {
FKSideInfo fkInfo = new FKSideInfo(leftRowCount,
leftNDV);
+ double pkSelectivity = pkSelectivity(joinRel, false, right, rightRowCount);
PKSideInfo pkInfo = new PKSideInfo(rightRowCount,
rightNDV,
joinRel.getJoinType().generatesNullsOnLeft() ? 1.0 :
- isPKSideSimpleTree ? RelMetadataQuery.getSelectivity(right, rightPred) : 1.0);
+ pkSelectivity);
return new PKFKRelationInfo(1, fkInfo, pkInfo, ndvScalingFactor, isPKSideSimpleTree);
}
@@ -292,6 +296,23 @@ public class HiveRelMdRowCount extends R
return null;
}
+ private static double pkSelectivity(JoinRelBase joinRel, boolean leftChild,
+ RelNode child,
+ double childRowCount) {
+ if ((leftChild && joinRel.getJoinType().generatesNullsOnRight()) ||
+ (!leftChild && joinRel.getJoinType().generatesNullsOnLeft())) {
+ return 1.0;
+ } else {
+ HiveTableScanRel tScan = HiveRelMdUniqueKeys.getTableScan(child, true);
+ if (tScan != null) {
+ double tRowCount = RelMetadataQuery.getRowCount(tScan);
+ return childRowCount / tRowCount;
+ } else {
+ return 1.0;
+ }
+ }
+ }
+
private static boolean isKey(BitSet c, RelNode rel) {
boolean isKey = false;
Set<BitSet> keys = RelMetadataQuery.getUniqueKeys(rel);
@@ -384,6 +405,10 @@ public class HiveRelMdRowCount extends R
@Override
public void visit(RelNode node, int ordinal, RelNode parent) {
+ if (node instanceof HepRelVertex) {
+ node = ((HepRelVertex) node).getCurrentRel();
+ }
+
if (node instanceof TableAccessRelBase) {
simpleTree = true;
} else if (node instanceof ProjectRelBase) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java Thu Oct 30 16:22:33 2014
@@ -37,6 +37,7 @@ import org.eigenbase.rel.metadata.RelMet
import org.eigenbase.rel.metadata.RelMetadataQuery;
import org.eigenbase.rex.RexNode;
import org.eigenbase.rex.RexUtil;
+import org.eigenbase.util.Pair;
import com.google.common.collect.ImmutableMap;
@@ -67,7 +68,15 @@ public class HiveRelMdSelectivity extend
private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) {
double ndvCrossProduct = 1;
- RexNode combinedPredicate = getCombinedPredicateForJoin(j, predicate);
+ Pair<Boolean, RexNode> predInfo =
+ getCombinedPredicateForJoin(j, predicate);
+ if (!predInfo.getKey()) {
+ return
+ new FilterSelectivityEstimator(j).
+ estimateSelectivity(predInfo.getValue());
+ }
+
+ RexNode combinedPredicate = predInfo.getValue();
JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j,
combinedPredicate);
ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap
@@ -175,7 +184,14 @@ public class HiveRelMdSelectivity extend
return ndvCrossProduct;
}
- private RexNode getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPredicate) {
+ /**
+ *
+ * @param j
+ * @param additionalPredicate
+ * @return if predicate is the join condition return (true, joinCond)
+ * else return (false, minusPred)
+ */
+ private Pair<Boolean,RexNode> getCombinedPredicateForJoin(HiveJoinRel j, RexNode additionalPredicate) {
RexNode minusPred = RelMdUtil.minusPreds(j.getCluster().getRexBuilder(), additionalPredicate,
j.getCondition());
@@ -184,10 +200,10 @@ public class HiveRelMdSelectivity extend
minusList.add(j.getCondition());
minusList.add(minusPred);
- return RexUtil.composeConjunction(j.getCluster().getRexBuilder(), minusList, true);
+ return new Pair<Boolean,RexNode>(false, minusPred);
}
- return j.getCondition();
+ return new Pair<Boolean,RexNode>(true,j.getCondition());
}
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdUniqueKeys.java Thu Oct 30 16:22:33 2014
@@ -30,6 +30,7 @@ import net.hydromatic.optiq.util.BitSets
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.ProjectRelBase;
import org.eigenbase.rel.RelNode;
import org.eigenbase.rel.metadata.BuiltInMetadata;
@@ -37,6 +38,7 @@ import org.eigenbase.rel.metadata.Metada
import org.eigenbase.rel.metadata.ReflectiveRelMetadataProvider;
import org.eigenbase.rel.metadata.RelMdUniqueKeys;
import org.eigenbase.rel.metadata.RelMetadataProvider;
+import org.eigenbase.relopt.hep.HepRelVertex;
import org.eigenbase.rex.RexInputRef;
import org.eigenbase.rex.RexNode;
@@ -59,16 +61,15 @@ public class HiveRelMdUniqueKeys {
*/
public Set<BitSet> getUniqueKeys(ProjectRelBase rel, boolean ignoreNulls) {
- RelNode child = rel.getChild();
+ HiveTableScanRel tScan = getTableScan(rel.getChild(), false);
- if (!(child instanceof HiveTableScanRel)) {
+ if ( tScan == null ) {
Function<RelNode, Metadata> fn = RelMdUniqueKeys.SOURCE.apply(
rel.getClass(), BuiltInMetadata.UniqueKeys.class);
return ((BuiltInMetadata.UniqueKeys) fn.apply(rel))
.getUniqueKeys(ignoreNulls);
}
- HiveTableScanRel tScan = (HiveTableScanRel) child;
Map<Integer, Integer> posMap = new HashMap<Integer, Integer>();
int projectPos = 0;
int colStatsPos = 0;
@@ -112,4 +113,26 @@ public class HiveRelMdUniqueKeys {
return keys;
}
+ /*
+ * traverse a path of Filter, Projects to get to the TableScan.
+ * In case of Unique keys, stop if you reach a Project, it will be handled
+ * by the invocation on the Project.
+ * In case of getting the base rowCount of a Path, keep going past a Project.
+ */
+ static HiveTableScanRel getTableScan(RelNode r, boolean traverseProject) {
+
+ while (r != null && !(r instanceof HiveTableScanRel)) {
+ if (r instanceof HepRelVertex) {
+ r = ((HepRelVertex) r).getCurrentRel();
+ } else if (r instanceof FilterRelBase) {
+ r = ((FilterRelBase) r).getChild();
+ } else if (traverseProject && r instanceof ProjectRelBase) {
+ r = ((ProjectRelBase) r).getChild();
+ } else {
+ r = null;
+ }
+ }
+ return r == null ? null : (HiveTableScanRel) r;
+ }
+
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java Thu Oct 30 16:22:33 2014
@@ -136,6 +136,10 @@ class ASTBuilder {
}
static ASTNode literal(RexLiteral literal) {
+ return literal(literal, false);
+ }
+
+ static ASTNode literal(RexLiteral literal, boolean useTypeQualInLiteral) {
Object val = null;
int type = 0;
SqlTypeName sqlType = literal.getType().getSqlTypeName();
@@ -147,18 +151,33 @@ class ASTBuilder {
type = HiveParser.BigintLiteral;
break;
case TINYINT:
- val = literal.getValue3();
+ if (useTypeQualInLiteral) {
+ val = literal.getValue3() + "Y";
+ } else {
+ val = literal.getValue3();
+ }
type = HiveParser.TinyintLiteral;
break;
case SMALLINT:
- val = literal.getValue3();
+ if (useTypeQualInLiteral) {
+ val = literal.getValue3() + "S";
+ } else {
+ val = literal.getValue3();
+ }
type = HiveParser.SmallintLiteral;
break;
case INTEGER:
- case BIGINT:
val = literal.getValue3();
type = HiveParser.BigintLiteral;
break;
+ case BIGINT:
+ if (useTypeQualInLiteral) {
+ val = literal.getValue3() + "L";
+ } else {
+ val = literal.getValue3();
+ }
+ type = HiveParser.BigintLiteral;
+ break;
case DOUBLE:
val = literal.getValue3() + "D";
type = HiveParser.Number;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java Thu Oct 30 16:22:33 2014
@@ -25,6 +25,8 @@ import java.util.Map;
import net.hydromatic.optiq.util.BitSets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException;
import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable;
@@ -63,6 +65,7 @@ import org.eigenbase.sql.type.SqlTypeNam
import com.google.common.collect.Iterables;
public class ASTConverter {
+ private static final Log LOG = LogFactory.getLog(ASTConverter.class);
private RelNode root;
private HiveAST hiveAST;
@@ -149,8 +152,9 @@ public class ASTConverter {
int i = 0;
for (RexNode r : select.getChildExps()) {
- ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select
- .getRowType().getFieldNames().get(i++));
+ ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(
+ new RexVisitor(schema, r instanceof RexLiteral)),
+ select.getRowType().getFieldNames().get(i++));
b.add(selectExpr);
}
}
@@ -329,10 +333,16 @@ public class ASTConverter {
static class RexVisitor extends RexVisitorImpl<ASTNode> {
private final Schema schema;
+ private boolean useTypeQualInLiteral;
protected RexVisitor(Schema schema) {
+ this(schema, false);
+ }
+
+ protected RexVisitor(Schema schema, boolean useTypeQualInLiteral) {
super(true);
this.schema = schema;
+ this.useTypeQualInLiteral = useTypeQualInLiteral;
}
@Override
@@ -357,7 +367,7 @@ public class ASTConverter {
@Override
public ASTNode visitLiteral(RexLiteral literal) {
- return ASTBuilder.literal(literal);
+ return ASTBuilder.literal(literal, useTypeQualInLiteral);
}
private ASTNode getPSpecAST(RexWindow window) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/PlanModifierForASTConv.java Thu Oct 30 16:22:33 2014
@@ -22,13 +22,18 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil;
import org.apache.hadoop.hive.ql.optimizer.optiq.OptiqSemanticException;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveAggregateRel;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveProjectRel;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveSortRel;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.eigenbase.rel.AggregateCall;
import org.eigenbase.rel.AggregateRelBase;
+import org.eigenbase.rel.Aggregation;
import org.eigenbase.rel.EmptyRel;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.JoinRelBase;
@@ -40,31 +45,52 @@ import org.eigenbase.rel.SetOpRel;
import org.eigenbase.rel.SingleRel;
import org.eigenbase.rel.SortRel;
import org.eigenbase.rel.rules.MultiJoinRel;
+import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.relopt.hep.HepRelVertex;
import org.eigenbase.relopt.volcano.RelSubset;
import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
import org.eigenbase.rex.RexNode;
+import org.eigenbase.sql.SqlKind;
import org.eigenbase.util.Pair;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
public class PlanModifierForASTConv {
+ private static final Log LOG = LogFactory.getLog(PlanModifierForASTConv.class);
public static RelNode convertOpTree(RelNode rel, List<FieldSchema> resultSchema)
throws OptiqSemanticException {
RelNode newTopNode = rel;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Original plan for PlanModifier\n " + RelOptUtil.toString(newTopNode));
+ }
if (!(newTopNode instanceof ProjectRelBase) && !(newTopNode instanceof SortRel)) {
newTopNode = introduceDerivedTable(newTopNode);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Plan after top-level introduceDerivedTable\n "
+ + RelOptUtil.toString(newTopNode));
+ }
}
convertOpTree(newTopNode, (RelNode) null);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Plan after nested convertOpTree\n " + RelOptUtil.toString(newTopNode));
+ }
Pair<RelNode, RelNode> topSelparentPair = HiveOptiqUtil.getTopLevelSelect(newTopNode);
fixTopOBSchema(newTopNode, topSelparentPair, resultSchema);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Plan after fixTopOBSchema\n " + RelOptUtil.toString(newTopNode));
+ }
+
topSelparentPair = HiveOptiqUtil.getTopLevelSelect(newTopNode);
newTopNode = renameTopLevelSelectInResultSchema(newTopNode, topSelparentPair, resultSchema);
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Final plan after modifier\n " + RelOptUtil.toString(newTopNode));
+ }
return newTopNode;
}
@@ -108,8 +134,14 @@ public class PlanModifierForASTConv {
introduceDerivedTable(((HiveSortRel) rel).getChild(), rel);
}
} else if (rel instanceof HiveAggregateRel) {
+ RelNode newParent = parent;
if (!validGBParent(rel, parent)) {
- introduceDerivedTable(rel, parent);
+ newParent = introduceDerivedTable(rel, parent);
+ }
+ // check if groupby is empty and there is no other cols in aggr
+ // this should only happen when newParent is constant.
+ if (isEmptyGrpAggr(rel)) {
+ replaceEmptyGroupAggr(rel, newParent);
}
}
}
@@ -125,36 +157,54 @@ public class PlanModifierForASTConv {
private static void fixTopOBSchema(final RelNode rootRel,
Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema)
throws OptiqSemanticException {
- if (topSelparentPair.getKey() instanceof SortRel
- && HiveOptiqUtil.orderRelNode(topSelparentPair.getKey())) {
- HiveSortRel obRel = (HiveSortRel) topSelparentPair.getKey();
- ProjectRelBase obChild = (ProjectRelBase) topSelparentPair.getValue();
-
- if (obChild.getRowType().getFieldCount() > resultSchema.size()) {
- RelDataType rt = obChild.getRowType();
- Set<Integer> collationInputRefs = new HashSet(RelCollationImpl.ordinals(obRel
- .getCollation()));
- ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
- for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
- if (collationInputRefs.contains(i)) {
- inputRefToCallMapBldr.put(i, obChild.getChildExps().get(i));
- }
- }
-
- ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
- if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) == resultSchema
- .size()) {
- HiveProjectRel replacementProjectRel = HiveProjectRel.create(obChild.getChild(), obChild
- .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames()
- .subList(0, resultSchema.size()));
- obRel.replaceInput(0, replacementProjectRel);
- obRel.setInputRefToCallMap(inputRefToCallMap);
- } else {
- throw new OptiqSemanticException(
- "Result Schema didn't match Optiq Optimized Op Tree Schema");
- }
+ if (!(topSelparentPair.getKey() instanceof SortRel)
+ || !HiveOptiqUtil.orderRelNode(topSelparentPair.getKey())) {
+ return;
+ }
+ HiveSortRel obRel = (HiveSortRel) topSelparentPair.getKey();
+ ProjectRelBase obChild = (ProjectRelBase) topSelparentPair.getValue();
+ if (obChild.getRowType().getFieldCount() <= resultSchema.size()) {
+ return;
+ }
+
+ RelDataType rt = obChild.getRowType();
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ Set<Integer> collationInputRefs = new HashSet(
+ RelCollationImpl.ordinals(obRel.getCollation()));
+ ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
+ for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
+ if (collationInputRefs.contains(i)) {
+ inputRefToCallMapBldr.put(i, obChild.getChildExps().get(i));
}
}
+ ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
+
+ if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) {
+ LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size()));
+ throw new OptiqSemanticException("Result Schema didn't match Optimized Op Tree Schema");
+ }
+ // This removes order-by only expressions from the projections.
+ HiveProjectRel replacementProjectRel = HiveProjectRel.create(obChild.getChild(), obChild
+ .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames()
+ .subList(0, resultSchema.size()));
+ obRel.replaceInput(0, replacementProjectRel);
+ obRel.setInputRefToCallMap(inputRefToCallMap);
+ }
+
+ private static String generateInvalidSchemaMessage(ProjectRelBase topLevelProj,
+ List<FieldSchema> resultSchema, int fieldsForOB) {
+ String errorDesc = "Result Schema didn't match Optiq Optimized Op Tree; schema: ";
+ for (FieldSchema fs : resultSchema) {
+ errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], ";
+ }
+ errorDesc += " projection fields: ";
+ for (RexNode exp : topLevelProj.getChildExps()) {
+ errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], ";
+ }
+ if (fieldsForOB != 0) {
+ errorDesc += fieldsForOB + " fields removed due to ORDER BY ";
+ }
+ return errorDesc.substring(0, errorDesc.length() - 2);
}
private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel,
@@ -167,22 +217,18 @@ public class PlanModifierForASTConv {
// (limit)?(OB)?(ProjectRelBase)....
List<RexNode> rootChildExps = originalProjRel.getChildExps();
if (resultSchema.size() != rootChildExps.size()) {
- // this is a bug in Hive where for queries like select key,value,value
- // convertRowSchemaToResultSetSchema() only returns schema containing
- // key,value
- // Underlying issue is much deeper because it seems like RowResolver
- // itself doesnt have
- // those mappings. see limit_pushdown.q & limit_pushdown_negative.q
- // Till Hive issue is fixed, disable CBO for such queries.
- throw new OptiqSemanticException("Result Schema didn't match Optiq Optimized Op Tree Schema");
+ // Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now.
+ LOG.error(generateInvalidSchemaMessage(originalProjRel, resultSchema, 0));
+ throw new OptiqSemanticException("Result Schema didn't match Optimized Op Tree Schema");
}
List<String> newSelAliases = new ArrayList<String>();
String colAlias;
for (int i = 0; i < rootChildExps.size(); i++) {
colAlias = resultSchema.get(i).getName();
- if (colAlias.startsWith("_"))
+ if (colAlias.startsWith("_")) {
colAlias = colAlias.substring(1);
+ }
newSelAliases.add(colAlias);
}
@@ -206,7 +252,7 @@ public class PlanModifierForASTConv {
return select;
}
- private static void introduceDerivedTable(final RelNode rel, RelNode parent) {
+ private static RelNode introduceDerivedTable(final RelNode rel, RelNode parent) {
int i = 0;
int pos = -1;
List<RelNode> childList = parent.getInputs();
@@ -226,6 +272,8 @@ public class PlanModifierForASTConv {
RelNode select = introduceDerivedTable(rel);
parent.replaceInput(pos, select);
+
+ return select;
}
private static boolean validJoinParent(RelNode joinNode, RelNode parent) {
@@ -310,4 +358,40 @@ public class PlanModifierForASTConv {
return validChild;
}
+
+ private static boolean isEmptyGrpAggr(RelNode gbNode) {
+ // Verify if both groupset and aggrfunction are empty)
+ AggregateRelBase aggrnode = (AggregateRelBase) gbNode;
+ if (aggrnode.getGroupSet().isEmpty() && aggrnode.getAggCallList().isEmpty()) {
+ return true;
+ }
+ return false;
+ }
+
+ private static void replaceEmptyGroupAggr(final RelNode rel, RelNode parent) {
+ // If this function is called, the parent should only include constant
+ List<RexNode> exps = parent.getChildExps();
+ for (RexNode rexNode : exps) {
+ if (rexNode.getKind() != SqlKind.LITERAL) {
+ throw new RuntimeException("We expect " + parent.toString()
+ + " to contain only constants. However, " + rexNode.toString() + " is "
+ + rexNode.getKind());
+ }
+ }
+ HiveAggregateRel oldAggRel = (HiveAggregateRel) rel;
+ RelDataTypeFactory typeFactory = oldAggRel.getCluster().getTypeFactory();
+ RelDataType longType = TypeConverter.convert(TypeInfoFactory.longTypeInfo, typeFactory);
+ RelDataType intType = TypeConverter.convert(TypeInfoFactory.intTypeInfo, typeFactory);
+ // Create the dummy aggregation.
+ Aggregation countFn = (Aggregation) SqlFunctionConverter.getOptiqAggFn("count",
+ ImmutableList.of(intType), longType);
+ // TODO: Using 0 might be wrong; might need to walk down to find the
+ // proper index of a dummy.
+ List<Integer> argList = ImmutableList.of(0);
+ AggregateCall dummyCall = new AggregateCall(countFn, false, argList, longType, null);
+ AggregateRelBase newAggRel = oldAggRel.copy(oldAggRel.getTraitSet(), oldAggRel.getChild(),
+ oldAggRel.getGroupSet(), ImmutableList.of(dummyCall));
+ RelNode select = introduceDerivedTable(newAggRel);
+ parent.replaceInput(0, select);
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Thu Oct 30 16:22:33 2014
@@ -82,35 +82,35 @@ public class RexNodeConverter {
private static final Log LOG = LogFactory.getLog(RexNodeConverter.class);
private static class InputCtx {
- private final RelDataType m_optiqInpDataType;
- private final ImmutableMap<String, Integer> m_hiveNameToPosMap;
- private final RowResolver m_hiveRR;
- private final int m_offsetInOptiqSchema;
+ private final RelDataType optiqInpDataType;
+ private final ImmutableMap<String, Integer> hiveNameToPosMap;
+ private final RowResolver hiveRR;
+ private final int offsetInOptiqSchema;
private InputCtx(RelDataType optiqInpDataType, ImmutableMap<String, Integer> hiveNameToPosMap,
RowResolver hiveRR, int offsetInOptiqSchema) {
- m_optiqInpDataType = optiqInpDataType;
- m_hiveNameToPosMap = hiveNameToPosMap;
- m_hiveRR = hiveRR;
- m_offsetInOptiqSchema = offsetInOptiqSchema;
+ this.optiqInpDataType = optiqInpDataType;
+ this.hiveNameToPosMap = hiveNameToPosMap;
+ this.hiveRR = hiveRR;
+ this.offsetInOptiqSchema = offsetInOptiqSchema;
}
};
- private final RelOptCluster m_cluster;
- private final ImmutableList<InputCtx> m_inputCtxs;
- private final boolean m_flattenExpr;
+ private final RelOptCluster cluster;
+ private final ImmutableList<InputCtx> inputCtxs;
+ private final boolean flattenExpr;
public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
ImmutableMap<String, Integer> nameToPosMap, int offset, boolean flattenExpr) {
- this.m_cluster = cluster;
- m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset));
- m_flattenExpr = flattenExpr;
+ this.cluster = cluster;
+ this.inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset));
+ this.flattenExpr = flattenExpr;
}
public RexNodeConverter(RelOptCluster cluster, List<InputCtx> inpCtxLst, boolean flattenExpr) {
- this.m_cluster = cluster;
- m_inputCtxs = ImmutableList.<InputCtx> builder().addAll(inpCtxLst).build();
- m_flattenExpr = flattenExpr;
+ this.cluster = cluster;
+ this.inputCtxs = ImmutableList.<InputCtx> builder().addAll(inpCtxLst).build();
+ this.flattenExpr = flattenExpr;
}
public RexNode convert(ExprNodeDesc expr) throws SemanticException {
@@ -134,7 +134,7 @@ public class RexNodeConverter {
RexNode rexNode = convert(fieldDesc.getDesc());
if (rexNode instanceof RexCall) {
// regular case of accessing nested field in a column
- return m_cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true);
+ return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true);
} else {
// This may happen for schema-less tables, where columns are dynamically
// supplied by serdes.
@@ -184,7 +184,7 @@ public class RexNodeConverter {
}
}
- argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), m_cluster.getTypeFactory()));
+ argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory()));
tmpRN = convert(tmpExprNode);
childRexNodeLst.add(tmpRN);
}
@@ -196,20 +196,20 @@ public class RexNodeConverter {
if (expr == null) {
// This is not a cast; process the function.
- retType = TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory());
+ retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory());
SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getFuncText(),
func.getGenericUDF(), argTypeBldr.build(), retType);
- expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
+ expr = cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
} else {
retType = expr.getType();
}
// TODO: Cast Function in Optiq have a bug where it infertype on cast throws
// an exception
- if (m_flattenExpr && (expr instanceof RexCall)
+ if (flattenExpr && (expr instanceof RexCall)
&& !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) {
RexCall call = (RexCall) expr;
- expr = m_cluster.getRexBuilder().makeCall(retType, call.getOperator(),
+ expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(),
RexUtil.flatten(call.getOperands(), call.getOperator()));
}
@@ -246,8 +246,8 @@ public class RexNodeConverter {
if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar)
|| (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate)
|| (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) {
- castExpr = m_cluster.getRexBuilder().makeAbstractCast(
- TypeConverter.convert(func.getTypeInfo(), m_cluster.getTypeFactory()),
+ castExpr = cluster.getRexBuilder().makeAbstractCast(
+ TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()),
childRexNodeLst.get(0));
}
}
@@ -258,15 +258,15 @@ public class RexNodeConverter {
private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException {
InputCtx ctxLookingFor = null;
- if (m_inputCtxs.size() == 1) {
- ctxLookingFor = m_inputCtxs.get(0);
+ if (inputCtxs.size() == 1) {
+ ctxLookingFor = inputCtxs.get(0);
} else {
String tableAlias = col.getTabAlias();
String colAlias = col.getColumn();
int noInp = 0;
- for (InputCtx ic : m_inputCtxs) {
- if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) {
- if (ic.m_hiveRR.getPosition(colAlias) >= 0) {
+ for (InputCtx ic : inputCtxs) {
+ if (tableAlias == null || ic.hiveRR.hasTableAlias(tableAlias)) {
+ if (ic.hiveRR.getPosition(colAlias) >= 0) {
ctxLookingFor = ic;
noInp++;
}
@@ -282,16 +282,16 @@ public class RexNodeConverter {
protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException {
InputCtx ic = getInputCtx(col);
- int pos = ic.m_hiveNameToPosMap.get(col.getColumn());
- return m_cluster.getRexBuilder().makeInputRef(
- ic.m_optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.m_offsetInOptiqSchema);
+ int pos = ic.hiveNameToPosMap.get(col.getColumn());
+ return cluster.getRexBuilder().makeInputRef(
+ ic.optiqInpDataType.getFieldList().get(pos).getType(), pos + ic.offsetInOptiqSchema);
}
private static final BigInteger MIN_LONG_BI = BigInteger.valueOf(Long.MIN_VALUE),
MAX_LONG_BI = BigInteger.valueOf(Long.MAX_VALUE);
protected RexNode convert(ExprNodeConstantDesc literal) throws OptiqSemanticException {
- RexBuilder rexBuilder = m_cluster.getRexBuilder();
+ RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
PrimitiveTypeInfo hiveType = (PrimitiveTypeInfo) literal.getTypeInfo();
RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory);
@@ -314,7 +314,7 @@ public class RexNodeConverter {
optiqLiteral = rexBuilder.makeBinaryLiteral(bs);
break;
case SHORT:
- optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value));
+ optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value), optiqDataType);
break;
case INT:
optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
@@ -353,7 +353,7 @@ public class RexNodeConverter {
// will work...
// An alternative would be to throw CboSemanticException and fall back
// to no CBO.
- RelDataType relType = m_cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL,
+ RelDataType relType = cluster.getTypeFactory().createSqlType(SqlTypeName.DECIMAL,
bd.scale(), unscaled.toString().length());
optiqLiteral = rexBuilder.makeExactLiteral(bd, relType);
}
@@ -397,8 +397,8 @@ public class RexNodeConverter {
}
private RexNode createNullLiteral(ExprNodeDesc expr) throws OptiqSemanticException {
- return m_cluster.getRexBuilder().makeNullLiteral(
- TypeConverter.convert(expr.getTypeInfo(), m_cluster.getTypeFactory()).getSqlTypeName());
+ return cluster.getRexBuilder().makeNullLiteral(
+ TypeConverter.convert(expr.getTypeInfo(), cluster.getTypeFactory()).getSqlTypeName());
}
public static RexNode convert(RelOptCluster cluster, ExprNodeDesc joinCondnExprNode,
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Thu Oct 30 16:22:33 2014
@@ -64,7 +64,7 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Maps;
public class SqlFunctionConverter {
- private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
+ private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
static final Map<String, SqlOperator> hiveToOptiq;
static final Map<SqlOperator, HiveToken> optiqToHiveToken;
@@ -87,10 +87,12 @@ public class SqlFunctionConverter {
} // do generic lookup
String name = null;
if (StringUtils.isEmpty(funcTextName)) {
- name = getName(hiveUDF); // this should probably never happen, see getName comment
+ name = getName(hiveUDF); // this should probably never happen, see getName
+ // comment
LOG.warn("The function text was empty, name from annotation is " + name);
} else {
- // We could just do toLowerCase here and let SA qualify it, but let's be proper...
+ // We could just do toLowerCase here and let SA qualify it, but let's be
+ // proper...
name = FunctionRegistry.getNormalizedFunctionName(funcTextName);
}
return getOptiqFn(name, optiqArgTypes, retType);
@@ -115,11 +117,10 @@ public class SqlFunctionConverter {
if (castType.equals(TypeInfoFactory.byteTypeInfo)) {
castUDF = FunctionRegistry.getFunctionInfo("tinyint");
} else if (castType instanceof CharTypeInfo) {
- castUDF = handleCastForParameterizedType(castType,
- FunctionRegistry.getFunctionInfo("char"));
+ castUDF = handleCastForParameterizedType(castType, FunctionRegistry.getFunctionInfo("char"));
} else if (castType instanceof VarcharTypeInfo) {
castUDF = handleCastForParameterizedType(castType,
- FunctionRegistry.getFunctionInfo("varchar"));
+ FunctionRegistry.getFunctionInfo("varchar"));
} else if (castType.equals(TypeInfoFactory.stringTypeInfo)) {
castUDF = FunctionRegistry.getFunctionInfo("string");
} else if (castType.equals(TypeInfoFactory.booleanTypeInfo)) {
@@ -140,24 +141,24 @@ public class SqlFunctionConverter {
castUDF = FunctionRegistry.getFunctionInfo("datetime");
} else if (castType instanceof DecimalTypeInfo) {
castUDF = handleCastForParameterizedType(castType,
- FunctionRegistry.getFunctionInfo("decimal"));
+ FunctionRegistry.getFunctionInfo("decimal"));
} else if (castType.equals(TypeInfoFactory.binaryTypeInfo)) {
castUDF = FunctionRegistry.getFunctionInfo("binary");
- } else throw new IllegalStateException("Unexpected type : " +
- castType.getQualifiedName());
+ } else
+ throw new IllegalStateException("Unexpected type : " + castType.getQualifiedName());
}
return castUDF;
}
private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, FunctionInfo fi) {
- SettableUDF udf = (SettableUDF)fi.getGenericUDF();
+ SettableUDF udf = (SettableUDF) fi.getGenericUDF();
try {
udf.setTypeInfo(ti);
} catch (UDFArgumentException e) {
throw new RuntimeException(e);
}
- return new FunctionInfo(fi.isNative(),fi.getDisplayName(),(GenericUDF)udf);
+ return new FunctionInfo(fi.isNative(), fi.getDisplayName(), (GenericUDF) udf);
}
// TODO: 1) handle Agg Func Name translation 2) is it correct to add func args
@@ -175,11 +176,10 @@ public class SqlFunctionConverter {
} else if (op.kind == SqlKind.PLUS_PREFIX) {
node = (ASTNode) ParseDriver.adaptor.create(HiveParser.PLUS, "PLUS");
} else {
- if (op.getName().toUpperCase()
- .equals(SqlStdOperatorTable.COUNT.getName())
+ if (op.getName().toUpperCase().equals(SqlStdOperatorTable.COUNT.getName())
&& children.size() == 0) {
- node = (ASTNode) ParseDriver.adaptor.create(
- HiveParser.TOK_FUNCTIONSTAR, "TOK_FUNCTIONSTAR");
+ node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTIONSTAR,
+ "TOK_FUNCTIONSTAR");
}
node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName()));
}
@@ -210,9 +210,12 @@ public class SqlFunctionConverter {
}
- // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry,
- // and only happen to match annotations. For user UDFs, the name is what user specifies at
- // creation time (annotation can be absent, different, or duplicate some other function).
+ // TODO: this is not valid. Function names for built-in UDFs are specified in
+ // FunctionRegistry,
+ // and only happen to match annotations. For user UDFs, the name is what user
+ // specifies at
+ // creation time (annotation can be absent, different, or duplicate some other
+ // function).
private static String getName(GenericUDF hiveUDF) {
String udfName = null;
if (hiveUDF instanceof GenericUDFBridge) {
@@ -287,70 +290,72 @@ public class SqlFunctionConverter {
}
public static class OptiqUDAF extends SqlAggFunction {
- final ImmutableList<RelDataType> m_argTypes;
- final RelDataType m_retType;
+ final ImmutableList<RelDataType> argTypes;
+ final RelDataType retType;
public OptiqUDAF(String opName, SqlReturnTypeInference returnTypeInference,
SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker,
ImmutableList<RelDataType> argTypes, RelDataType retType) {
super(opName, SqlKind.OTHER_FUNCTION, returnTypeInference, operandTypeInference,
operandTypeChecker, SqlFunctionCategory.USER_DEFINED_FUNCTION);
- m_argTypes = argTypes;
- m_retType = retType;
+ this.argTypes = argTypes;
+ this.retType = retType;
}
@Override
public List<RelDataType> getParameterTypes(final RelDataTypeFactory typeFactory) {
- return m_argTypes;
+ return this.argTypes;
}
@Override
public RelDataType getReturnType(final RelDataTypeFactory typeFactory) {
- return m_retType;
+ return this.retType;
}
}
private static class OptiqUDFInfo {
- private String m_udfName;
- private SqlReturnTypeInference m_returnTypeInference;
- private SqlOperandTypeInference m_operandTypeInference;
- private SqlOperandTypeChecker m_operandTypeChecker;
- private ImmutableList<RelDataType> m_argTypes;
- private RelDataType m_retType;
+ private String udfName;
+ private SqlReturnTypeInference returnTypeInference;
+ private SqlOperandTypeInference operandTypeInference;
+ private SqlOperandTypeChecker operandTypeChecker;
+ private ImmutableList<RelDataType> argTypes;
+ private RelDataType retType;
}
private static OptiqUDFInfo getUDFInfo(String hiveUdfName,
ImmutableList<RelDataType> optiqArgTypes, RelDataType optiqRetType) {
OptiqUDFInfo udfInfo = new OptiqUDFInfo();
- udfInfo.m_udfName = hiveUdfName;
- udfInfo.m_returnTypeInference = ReturnTypes.explicit(optiqRetType);
- udfInfo.m_operandTypeInference = InferTypes.explicit(optiqArgTypes);
+ udfInfo.udfName = hiveUdfName;
+ udfInfo.returnTypeInference = ReturnTypes.explicit(optiqRetType);
+ udfInfo.operandTypeInference = InferTypes.explicit(optiqArgTypes);
ImmutableList.Builder<SqlTypeFamily> typeFamilyBuilder = new ImmutableList.Builder<SqlTypeFamily>();
for (RelDataType at : optiqArgTypes) {
typeFamilyBuilder.add(Util.first(at.getSqlTypeName().getFamily(), SqlTypeFamily.ANY));
}
- udfInfo.m_operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build());
+ udfInfo.operandTypeChecker = OperandTypes.family(typeFamilyBuilder.build());
- udfInfo.m_argTypes = ImmutableList.<RelDataType> copyOf(optiqArgTypes);
- udfInfo.m_retType = optiqRetType;
+ udfInfo.argTypes = ImmutableList.<RelDataType> copyOf(optiqArgTypes);
+ udfInfo.retType = optiqRetType;
return udfInfo;
}
public static SqlOperator getOptiqFn(String hiveUdfName,
- ImmutableList<RelDataType> optiqArgTypes, RelDataType optiqRetType) throws OptiqSemanticException{
+ ImmutableList<RelDataType> optiqArgTypes, RelDataType optiqRetType)
+ throws OptiqSemanticException {
if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) {
// We can create Optiq IS_DISTINCT_FROM operator for this. But since our
- // join reordering algo cant handle this anyway there is no advantage of this.
+ // join reordering algo cant handle this anyway there is no advantage of
+ // this.
// So, bail out for now.
throw new OptiqSemanticException("<=> is not yet supported for cbo.");
}
SqlOperator optiqOp = hiveToOptiq.get(hiveUdfName);
if (optiqOp == null) {
OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType);
- optiqOp = new SqlFunction(uInf.m_udfName, SqlKind.OTHER_FUNCTION, uInf.m_returnTypeInference,
- uInf.m_operandTypeInference, uInf.m_operandTypeChecker,
+ optiqOp = new SqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference,
+ uInf.operandTypeInference, uInf.operandTypeChecker,
SqlFunctionCategory.USER_DEFINED_FUNCTION);
}
@@ -363,8 +368,8 @@ public class SqlFunctionConverter {
if (optiqAggFn == null) {
OptiqUDFInfo uInf = getUDFInfo(hiveUdfName, optiqArgTypes, optiqRetType);
- optiqAggFn = new OptiqUDAF(uInf.m_udfName, uInf.m_returnTypeInference,
- uInf.m_operandTypeInference, uInf.m_operandTypeChecker, uInf.m_argTypes, uInf.m_retType);
+ optiqAggFn = new OptiqUDAF(uInf.udfName, uInf.returnTypeInference, uInf.operandTypeInference,
+ uInf.operandTypeChecker, uInf.argTypes, uInf.retType);
}
return optiqAggFn;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java?rev=1635536&r1=1635535&r2=1635536&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java Thu Oct 30 16:22:33 2014
@@ -31,10 +31,13 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
@@ -133,7 +136,15 @@ public final class PcrOpProcFactory {
if (wrapper.state == PcrExprProcFactory.WalkState.TRUE) {
owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop));
- } else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) {
+ } else if (wrapper.state == PcrExprProcFactory.WalkState.CONSTANT && wrapper.outExpr instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeDesc desc = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)wrapper.outExpr);
+ if (desc != null && desc instanceof ExprNodeConstantDesc && Boolean.TRUE.equals(((ExprNodeConstantDesc)desc).getValue())) {
+ owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop));
+ } else {
+ fop.getConf().setPredicate(wrapper.outExpr);
+ }
+ }
+ else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) {
fop.getConf().setPredicate(wrapper.outExpr);
} else {
LOG.warn("Filter passes no row");