You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2016/01/27 04:08:56 UTC
hive git commit: HIVE-12805 : CBO: Calcite Operator To Hive Operator
(Calcite Return Path): MiniTezCliDriver skewjoin.q failure(Hari Subramaniyan,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 27172bcb4 -> bdf20ef57
HIVE-12805 : CBO: Calcite Operator To Hive Operator (Calcite Return Path): MiniTezCliDriver skewjoin.q failure(Hari Subramaniyan, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bdf20ef5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bdf20ef5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bdf20ef5
Branch: refs/heads/master
Commit: bdf20ef5709b62471c1c6cd2a0c317791465c7c1
Parents: 27172bc
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Tue Jan 26 19:08:45 2016 -0800
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Tue Jan 26 19:08:45 2016 -0800
----------------------------------------------------------------------
.../calcite/reloperators/HiveMultiJoin.java | 29 +++++++++++----
.../rules/HiveInsertExchange4JoinRule.java | 10 ++---
.../calcite/rules/HiveJoinToMultiJoinRule.java | 39 +++++++++++++++++++-
3 files changed, 64 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
index 660f01d..cff737c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
@@ -70,6 +70,7 @@ public final class HiveMultiJoin extends AbstractRelNode {
* INNER
* @param filters filters associated with each join
* input
+ * @param joinPredicateInfo join predicate information
*/
public HiveMultiJoin(
RelOptCluster cluster,
@@ -78,9 +79,10 @@ public final class HiveMultiJoin extends AbstractRelNode {
RelDataType rowType,
List<Pair<Integer,Integer>> joinInputs,
List<JoinRelType> joinTypes,
- List<RexNode> filters) {
+ List<RexNode> filters,
+ JoinPredicateInfo joinPredicateInfo) {
super(cluster, TraitsUtil.getDefaultTraitSet(cluster));
- this.inputs = Lists.newArrayList(inputs);
+ this.inputs = inputs;
this.condition = condition;
this.rowType = rowType;
@@ -89,14 +91,27 @@ public final class HiveMultiJoin extends AbstractRelNode {
this.joinTypes = ImmutableList.copyOf(joinTypes);
this.filters = ImmutableList.copyOf(filters);
this.outerJoin = containsOuter();
-
- try {
- this.joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(this);
- } catch (CalciteSemanticException e) {
- throw new RuntimeException(e);
+ if (joinPredicateInfo == null) {
+ try {
+ this.joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(this);
+ } catch (CalciteSemanticException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ this.joinPredInfo = joinPredicateInfo;
}
}
+ public HiveMultiJoin(
+ RelOptCluster cluster,
+ List<RelNode> inputs,
+ RexNode condition,
+ RelDataType rowType,
+ List<Pair<Integer,Integer>> joinInputs,
+ List<JoinRelType> joinTypes,
+ List<RexNode> filters) {
+ this(cluster, Lists.newArrayList(inputs), condition, rowType, joinInputs, joinTypes, filters, null);
+ }
@Override
public void replaceInput(int ordinalInParent, RelNode p) {
http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
index d6e3915..c9cf396 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinLeafPredi
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelCollation;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
@@ -76,11 +77,10 @@ public class HiveInsertExchange4JoinRule extends RelOptRule {
JoinPredicateInfo joinPredInfo;
if (call.rel(0) instanceof HiveMultiJoin) {
HiveMultiJoin multiJoin = call.rel(0);
- try {
- joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(multiJoin);
- } catch (CalciteSemanticException e) {
- throw new RuntimeException(e);
- }
+ joinPredInfo = multiJoin.getJoinPredicateInfo();
+ } else if (call.rel(0) instanceof HiveJoin) {
+ HiveJoin hiveJoin = call.rel(0);
+ joinPredInfo = hiveJoin.getJoinPredicateInfo();
} else if (call.rel(0) instanceof Join) {
Join join = call.rel(0);
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
index 5d169a1..ce9535f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
@@ -239,14 +239,49 @@ public class HiveJoinToMultiJoinRule extends RelOptRule {
// We can now create a multijoin operator
RexNode newCondition = RexUtil.flatten(rexBuilder,
RexUtil.composeConjunction(rexBuilder, newJoinCondition, false));
+ List<RelNode> newInputsArray = Lists.newArrayList(newInputs);
+ JoinPredicateInfo joinPredInfo = null;
+ try {
+ joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(newInputsArray, systemFieldList, newCondition);
+ } catch (CalciteSemanticException e) {
+ throw new RuntimeException(e);
+ }
+
+ // If the number of joins < number of input tables-1, this is not a star join.
+ if (joinPredInfo.getEquiJoinPredicateElements().size() < newInputs.size()-1) {
+ return null;
+ }
+ // Validate that the multi-join is a valid star join before returning it.
+ for (int i=0; i<newInputs.size(); i++) {
+ List<RexNode> joinKeys = null;
+ for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
+ List<RexNode> currJoinKeys = joinPredInfo.
+ getEquiJoinPredicateElements().get(j).getJoinExprs(i);
+ if (currJoinKeys.isEmpty()) {
+ continue;
+ }
+ if (joinKeys == null) {
+ joinKeys = currJoinKeys;
+ } else {
+ // If we join on different keys on different tables, we can no longer apply
+ // multi-join conversion as this is no longer a valid star join.
+ // Bail out if this is the case.
+ if (!joinKeys.containsAll(currJoinKeys) || !currJoinKeys.containsAll(joinKeys)) {
+ return null;
+ }
+ }
+ }
+ }
+
return new HiveMultiJoin(
join.getCluster(),
- newInputs,
+ newInputsArray,
newCondition,
join.getRowType(),
joinInputs,
joinTypes,
- joinFilters);
+ joinFilters,
+ joinPredInfo);
}
/*