You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/10/31 15:10:54 UTC
svn commit: r1537500 [1/2] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/ test/org/apache/hadoop/hive/ql/parse/
test/queries/clientpositive/ test/results/clientpositive/
Author: hashutosh
Date: Thu Oct 31 14:10:54 2013
New Revision: 1537500
URL: http://svn.apache.org/r1537500
Log:
HIVE-5556 : Pushdown join conditions (Harish Butani via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBJoinTreeApplyPredicate.java
hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_1.q
hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_2.q
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out
hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java?rev=1537500&r1=1537499&r2=1537500&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java Thu Oct 31 14:10:54 2013
@@ -111,7 +111,11 @@ public class QBJoinTree implements Seria
* String
*/
public void setLeftAlias(String leftAlias) {
- this.leftAlias = leftAlias;
+ if ( this.leftAlias != null && !this.leftAlias.equals(leftAlias) ) {
+ this.leftAlias = null;
+ } else {
+ this.leftAlias = leftAlias;
+ }
}
public String[] getRightAliases() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1537500&r1=1537499&r2=1537500&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Oct 31 14:10:54 2013
@@ -1379,7 +1379,7 @@ public class SemanticAnalyzer extends Ba
}
@SuppressWarnings("nls")
- private void parseJoinCondPopulateAlias(QBJoinTree joinTree, ASTNode condn,
+ void parseJoinCondPopulateAlias(QBJoinTree joinTree, ASTNode condn,
ArrayList<String> leftAliases, ArrayList<String> rightAliases,
ArrayList<String> fields) throws SemanticException {
// String[] allAliases = joinTree.getAllAliases();
@@ -1501,6 +1501,160 @@ public class SemanticAnalyzer extends Ba
}
}
+ /*
+ * refactored out of the Equality case of parseJoinCondition
+ * so that this can be recursively called on its left tree in the case when
+ * only left sources are referenced in a Predicate
+ */
+ void applyEqualityPredicateToQBJoinTree(QBJoinTree joinTree,
+ JoinType type,
+ List<String> leftSrc,
+ ASTNode joinCond,
+ ASTNode leftCondn,
+ ASTNode rightCondn,
+ List<String> leftCondAl1,
+ List<String> leftCondAl2,
+ List<String> rightCondAl1,
+ List<String> rightCondAl2) throws SemanticException {
+ if (leftCondAl1.size() != 0) {
+ if ((rightCondAl1.size() != 0)
+ || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
+ if (type.equals(JoinType.LEFTOUTER) ||
+ type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(0).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ /*
+ * If the rhs references table sources and this QBJoinTree has a leftTree;
+ * hand it to the leftTree and let it recursively handle it.
+ * There are 3 cases of passing a condition down:
+ * 1. The leftSide && rightSide don't contains references to the leftTree's rightAlias
+ * => pass the lists down as is.
+ * 2. The leftSide contains refs to the leftTree's rightAlias, the rightSide doesn't
+ * => switch the leftCondAl1 and leftConAl2 lists and pass down.
+ * 3. The rightSide contains refs to the leftTree's rightAlias, the leftSide doesn't
+ * => switch the rightCondAl1 and rightConAl2 lists and pass down.
+ * 4. In case both contain references to the leftTree's rightAlias
+ * => we cannot push the condition down.
+ * 5. If either contain references to both left & right
+ * => we cannot push forward.
+ */
+ if (rightCondAl1.size() != 0) {
+ QBJoinTree leftTree = joinTree.getJoinSrc();
+ List<String> leftTreeLeftSrc = new ArrayList<String>();
+ if (leftTree != null) {
+ String leftTreeRightSource = leftTree.getRightAliases() != null &&
+ leftTree.getRightAliases().length > 0 ?
+ leftTree.getRightAliases()[0] : null;
+
+ boolean leftHasRightReference = false;
+ for (String r : leftCondAl1) {
+ if (r.equals(leftTreeRightSource)) {
+ leftHasRightReference = true;
+ break;
+ }
+ }
+ boolean rightHasRightReference = false;
+ for (String r : rightCondAl1) {
+ if (r.equals(leftTreeRightSource)) {
+ rightHasRightReference = true;
+ break;
+ }
+ }
+
+ boolean pushedDown = false;
+ if ( !leftHasRightReference && !rightHasRightReference ) {
+ applyEqualityPredicateToQBJoinTree(leftTree, type, leftTreeLeftSrc,
+ joinCond, leftCondn, rightCondn,
+ leftCondAl1, leftCondAl2,
+ rightCondAl1, rightCondAl2);
+ pushedDown = true;
+ } else if ( !leftHasRightReference && rightHasRightReference && rightCondAl1.size() == 1 ) {
+ applyEqualityPredicateToQBJoinTree(leftTree, type, leftTreeLeftSrc,
+ joinCond, leftCondn, rightCondn,
+ leftCondAl1, leftCondAl2,
+ rightCondAl2, rightCondAl1);
+ pushedDown = true;
+ } else if (leftHasRightReference && !rightHasRightReference && leftCondAl1.size() == 1 ) {
+ applyEqualityPredicateToQBJoinTree(leftTree, type, leftTreeLeftSrc,
+ joinCond, leftCondn, rightCondn,
+ leftCondAl2, leftCondAl1,
+ rightCondAl1, rightCondAl2);
+ pushedDown = true;
+ }
+
+ if (leftTreeLeftSrc.size() == 1) {
+ leftTree.setLeftAlias(leftTreeLeftSrc.get(0));
+ }
+ if ( pushedDown) {
+ return;
+ }
+ } // leftTree != null
+ }
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else if (rightCondAl2.size() != 0) {
+ populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
+ leftSrc);
+ populateAliases(rightCondAl1, rightCondAl2, rightCondn, joinTree,
+ leftSrc);
+ boolean nullsafe = joinCond.getToken().getType() == HiveParser.EQUAL_NS;
+ joinTree.getNullSafes().add(nullsafe);
+ }
+ } else if (leftCondAl2.size() != 0) {
+ if ((rightCondAl2.size() != 0)
+ || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
+ if (type.equals(JoinType.RIGHTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(1).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else if (rightCondAl1.size() != 0) {
+ populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
+ leftSrc);
+ populateAliases(rightCondAl1, rightCondAl2, rightCondn, joinTree,
+ leftSrc);
+ boolean nullsafe = joinCond.getToken().getType() == HiveParser.EQUAL_NS;
+ joinTree.getNullSafes().add(nullsafe);
+ }
+ } else if (rightCondAl1.size() != 0) {
+ if (type.equals(JoinType.LEFTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(0).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ if (type.equals(JoinType.RIGHTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(1).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ }
+
+ }
+
private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, List<String> leftSrc)
throws SemanticException {
if (joinCond == null) {
@@ -1581,75 +1735,10 @@ public class SemanticAnalyzer extends Ba
.getMsg(joinCond));
}
- if (leftCondAl1.size() != 0) {
- if ((rightCondAl1.size() != 0)
- || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
- if (type.equals(JoinType.LEFTOUTER) ||
- type.equals(JoinType.FULLOUTER)) {
- if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
- joinTree.getFilters().get(0).add(joinCond);
- } else {
- LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
- joinTree.getFiltersForPushing().get(0).add(joinCond);
- }
- } else {
- joinTree.getFiltersForPushing().get(0).add(joinCond);
- }
- } else if (rightCondAl2.size() != 0) {
- populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
- leftSrc);
- populateAliases(rightCondAl1, rightCondAl2, rightCondn, joinTree,
- leftSrc);
- boolean nullsafe = joinCond.getToken().getType() == HiveParser.EQUAL_NS;
- joinTree.getNullSafes().add(nullsafe);
- }
- } else if (leftCondAl2.size() != 0) {
- if ((rightCondAl2.size() != 0)
- || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
- if (type.equals(JoinType.RIGHTOUTER)
- || type.equals(JoinType.FULLOUTER)) {
- if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
- joinTree.getFilters().get(1).add(joinCond);
- } else {
- LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
- joinTree.getFiltersForPushing().get(1).add(joinCond);
- }
- } else {
- joinTree.getFiltersForPushing().get(1).add(joinCond);
- }
- } else if (rightCondAl1.size() != 0) {
- populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
- leftSrc);
- populateAliases(rightCondAl1, rightCondAl2, rightCondn, joinTree,
- leftSrc);
- boolean nullsafe = joinCond.getToken().getType() == HiveParser.EQUAL_NS;
- joinTree.getNullSafes().add(nullsafe);
- }
- } else if (rightCondAl1.size() != 0) {
- if (type.equals(JoinType.LEFTOUTER)
- || type.equals(JoinType.FULLOUTER)) {
- if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
- joinTree.getFilters().get(0).add(joinCond);
- } else {
- LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
- joinTree.getFiltersForPushing().get(0).add(joinCond);
- }
- } else {
- joinTree.getFiltersForPushing().get(0).add(joinCond);
- }
- } else {
- if (type.equals(JoinType.RIGHTOUTER)
- || type.equals(JoinType.FULLOUTER)) {
- if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
- joinTree.getFilters().get(1).add(joinCond);
- } else {
- LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
- joinTree.getFiltersForPushing().get(1).add(joinCond);
- }
- } else {
- joinTree.getFiltersForPushing().get(1).add(joinCond);
- }
- }
+ applyEqualityPredicateToQBJoinTree(joinTree, type, leftSrc,
+ joinCond, leftCondn, rightCondn,
+ leftCondAl1, leftCondAl2,
+ rightCondAl1, rightCondAl2);
break;
@@ -6848,7 +6937,7 @@ public class SemanticAnalyzer extends Ba
/**
* Merges node to target
*/
- private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos) {
+ private void mergeJoins(QB qb, QBJoinTree node, QBJoinTree target, int pos, int[] tgtToNodeExprMap) {
String[] nodeRightAliases = node.getRightAliases();
String[] trgtRightAliases = target.getRightAliases();
String[] rightAliases = new String[nodeRightAliases.length
@@ -6877,7 +6966,12 @@ public class SemanticAnalyzer extends Ba
ArrayList<ArrayList<ASTNode>> expr = target.getExpressions();
for (int i = 0; i < nodeRightAliases.length; i++) {
- expr.add(node.getExpressions().get(i + 1));
+ List<ASTNode> nodeConds = node.getExpressions().get(i + 1);
+ ArrayList<ASTNode> reordereNodeConds = new ArrayList<ASTNode>();
+ for(int k=0; k < tgtToNodeExprMap.length; k++) {
+ reordereNodeConds.add(nodeConds.get(k));
+ }
+ expr.add(reordereNodeConds);
}
ArrayList<Boolean> nns = node.getNullSafes();
@@ -6994,11 +7088,11 @@ public class SemanticAnalyzer extends Ba
}
}
- private int findMergePos(QBJoinTree node, QBJoinTree target) {
+ private ObjectPair<Integer, int[]> findMergePos(QBJoinTree node, QBJoinTree target) {
int res = -1;
String leftAlias = node.getLeftAlias();
if (leftAlias == null) {
- return -1;
+ return new ObjectPair(-1, null);
}
ArrayList<ASTNode> nodeCondn = node.getExpressions().get(0);
@@ -7017,18 +7111,41 @@ public class SemanticAnalyzer extends Ba
}
}
- if ((targetCondn == null) || (nodeCondn.size() != targetCondn.size())) {
- return -1;
+ if ( targetCondn == null ) {
+ return new ObjectPair(-1, null);
+ }
+
+ /*
+ * The order of the join condition expressions don't matter.
+ * A merge can happen:
+ * - if every target condition is present in some position of the node condition list.
+ * - there is no node condition, which is not equal to any target condition.
+ */
+
+ int[] tgtToNodeExprMap = new int[targetCondn.size()];
+ boolean[] nodeFiltersMapped = new boolean[nodeCondn.size()];
+ int i, j;
+ for(i=0; i<targetCondn.size(); i++) {
+ String tgtExprTree = targetCondn.get(i).toStringTree();
+ tgtToNodeExprMap[i] = -1;
+ for(j=0; j < nodeCondn.size(); j++) {
+ if ( nodeCondn.get(j).toStringTree().equals(tgtExprTree)) {
+ tgtToNodeExprMap[i] = j;
+ nodeFiltersMapped[j] = true;
+ }
+ }
+ if ( tgtToNodeExprMap[i] == -1) {
+ return new ObjectPair(-1, null);
+ }
}
- for (int i = 0; i < nodeCondn.size(); i++) {
- if (!nodeCondn.get(i).toStringTree().equals(
- targetCondn.get(i).toStringTree())) {
- return -1;
+ for(j=0; j < nodeCondn.size(); j++) {
+ if ( !nodeFiltersMapped[j]) {
+ return new ObjectPair(-1, null);
}
}
- return res;
+ return new ObjectPair(res, tgtToNodeExprMap);
}
// try merge join tree from inner most source
@@ -7063,7 +7180,8 @@ public class SemanticAnalyzer extends Ba
if (prevType != null && prevType != currType) {
break;
}
- int pos = findMergePos(node, target);
+ ObjectPair<Integer, int[]> mergeDetails = findMergePos(node, target);
+ int pos = mergeDetails.getFirst();
if (pos >= 0) {
// for outer joins, it should not exceed 16 aliases (short type)
if (!node.getNoOuterJoin() || !target.getNoOuterJoin()) {
@@ -7072,7 +7190,7 @@ public class SemanticAnalyzer extends Ba
continue;
}
}
- mergeJoins(qb, node, target, pos);
+ mergeJoins(qb, node, target, pos, mergeDetails.getSecond());
trees.set(j, null);
continue; // continue merging with next alias
}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBJoinTreeApplyPredicate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBJoinTreeApplyPredicate.java?rev=1537500&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBJoinTreeApplyPredicate.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBJoinTreeApplyPredicate.java Thu Oct 31 14:10:54 2013
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestQBJoinTreeApplyPredicate {
+
+ static HiveConf conf;
+
+ SemanticAnalyzer sA;
+
+ @BeforeClass
+ public static void initialize() {
+ conf = new HiveConf(SemanticAnalyzer.class);
+ SessionState.start(conf);
+ }
+
+ @Before
+ public void setup() throws SemanticException {
+ sA = new SemanticAnalyzer(conf);
+ }
+
+ static ASTNode constructIdentifier(String nm) {
+ return (ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, nm);
+ }
+
+ static ASTNode constructTabRef(String tblNm) {
+ ASTNode table = (ASTNode)
+ ParseDriver.adaptor.create(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL");
+ ASTNode id = constructIdentifier(tblNm);
+ table.addChild(id);
+ return table;
+ }
+
+ static ASTNode constructColRef(String tblNm, String colNm) {
+ ASTNode table = constructTabRef(tblNm);
+ ASTNode col = constructIdentifier(colNm);
+ ASTNode dot = (ASTNode) ParseDriver.adaptor.create(HiveParser.DOT, ".");
+ dot.addChild(table);
+ dot.addChild(col);
+ return dot;
+ }
+
+ static ASTNode constructEqualityCond(String lTbl, String lCol, String rTbl, String rCol) {
+ ASTNode lRef = constructColRef(lTbl, lCol);
+ ASTNode rRef = constructColRef(rTbl, rCol);
+ ASTNode eq = (ASTNode) ParseDriver.adaptor.create(HiveParser.EQUAL, "=");
+ eq.addChild(lRef);
+ eq.addChild(rRef);
+ return eq;
+ }
+
+ QBJoinTree createJoinTree(JoinType type,
+ String leftAlias,
+ QBJoinTree leftTree,
+ String rightAlias) {
+ QBJoinTree jT = new QBJoinTree();
+ JoinCond[] condn = new JoinCond[1];
+ condn[0] = new JoinCond(0, 1, type);
+ if ( leftTree == null ) {
+ jT.setLeftAlias(leftAlias);
+ String[] leftAliases = new String[1];
+ leftAliases[0] = leftAlias;
+ jT.setLeftAliases(leftAliases);
+ } else {
+ jT.setJoinSrc(leftTree);
+ String[] leftChildAliases = leftTree.getLeftAliases();
+ String leftAliases[] = new String[leftChildAliases.length + 1];
+ for (int i = 0; i < leftChildAliases.length; i++) {
+ leftAliases[i] = leftChildAliases[i];
+ }
+ leftAliases[leftChildAliases.length] = leftTree.getRightAliases()[0];
+ jT.setLeftAliases(leftAliases);
+ }
+ String[] rightAliases = new String[1];
+ rightAliases[0] = rightAlias;
+ jT.setRightAliases(rightAliases);
+ String[] children = new String[2];
+ children[0] = leftAlias;
+ children[1] = rightAlias;
+ jT.setBaseSrc(children);
+ ArrayList<ArrayList<ASTNode>> expressions = new ArrayList<ArrayList<ASTNode>>();
+ expressions.add(new ArrayList<ASTNode>());
+ expressions.add(new ArrayList<ASTNode>());
+ jT.setExpressions(expressions);
+
+ ArrayList<Boolean> nullsafes = new ArrayList<Boolean>();
+ jT.setNullSafes(nullsafes);
+
+ ArrayList<ArrayList<ASTNode>> filters = new ArrayList<ArrayList<ASTNode>>();
+ filters.add(new ArrayList<ASTNode>());
+ filters.add(new ArrayList<ASTNode>());
+ jT.setFilters(filters);
+ jT.setFilterMap(new int[2][]);
+
+ ArrayList<ArrayList<ASTNode>> filtersForPushing =
+ new ArrayList<ArrayList<ASTNode>>();
+ filtersForPushing.add(new ArrayList<ASTNode>());
+ filtersForPushing.add(new ArrayList<ASTNode>());
+ jT.setFiltersForPushing(filtersForPushing);
+
+ return jT;
+ }
+
+ ASTNode applyEqPredicate(QBJoinTree jT,
+ String lTbl, String lCol,
+ String rTbl, String rCol) throws SemanticException {
+ ASTNode joinCond = constructEqualityCond(lTbl, lCol, rTbl, rCol);
+
+ ASTNode leftCondn = (ASTNode) joinCond.getChild(0);
+ ASTNode rightCondn = (ASTNode) joinCond.getChild(1);
+
+ List<String> leftSrc = new ArrayList<String>();
+ ArrayList<String> leftCondAl1 = new ArrayList<String>();
+ ArrayList<String> leftCondAl2 = new ArrayList<String>();
+ ArrayList<String> rightCondAl1 = new ArrayList<String>();
+ ArrayList<String> rightCondAl2 = new ArrayList<String>();
+
+ sA.parseJoinCondPopulateAlias(jT, leftCondn, leftCondAl1, leftCondAl2, null);
+ sA.parseJoinCondPopulateAlias(jT, rightCondn, rightCondAl1, rightCondAl2, null);
+
+ sA.applyEqualityPredicateToQBJoinTree(jT, JoinType.INNER, leftSrc, joinCond,
+ leftCondn,
+ rightCondn,
+ leftCondAl1, leftCondAl2, rightCondAl1, rightCondAl2);
+ return joinCond;
+ }
+
+ @Test
+ public void testSimpleCondn() throws SemanticException {
+ QBJoinTree jT = createJoinTree(JoinType.INNER, "a", null, "b");
+ ASTNode joinCond = applyEqPredicate(jT, "a", "x", "b", "y");
+ Assert.assertEquals(jT.getExpressions().get(0).get(0), joinCond.getChild(0));
+ Assert.assertEquals(jT.getExpressions().get(1).get(0), joinCond.getChild(1));
+ }
+
+ @Test
+ public void test3WayJoin() throws SemanticException {
+ QBJoinTree jT1 = createJoinTree(JoinType.INNER, "a", null, "b");
+ QBJoinTree jT = createJoinTree(JoinType.INNER, "b", jT1, "c");
+ ASTNode joinCond1 = applyEqPredicate(jT, "a", "x", "b", "y");
+ ASTNode joinCond2 = applyEqPredicate(jT, "b", "y", "c", "z");
+ Assert.assertEquals(jT1.getExpressions().get(0).get(0), joinCond1.getChild(0));
+ Assert.assertEquals(jT1.getExpressions().get(1).get(0), joinCond1.getChild(1));
+ Assert.assertEquals(jT.getExpressions().get(0).get(0), joinCond2.getChild(0));
+ Assert.assertEquals(jT.getExpressions().get(1).get(0), joinCond2.getChild(1));
+ }
+
+ @Test
+ public void test3WayJoinSwitched() throws SemanticException {
+ QBJoinTree jT1 = createJoinTree(JoinType.INNER, "a", null, "b");
+ QBJoinTree jT = createJoinTree(JoinType.INNER, "b", jT1, "c");
+ ASTNode joinCond1 = applyEqPredicate(jT, "b", "y", "a", "x");
+ ASTNode joinCond2 = applyEqPredicate(jT, "b", "y", "c", "z");
+ Assert.assertEquals(jT1.getExpressions().get(0).get(0), joinCond1.getChild(1));
+ Assert.assertEquals(jT1.getExpressions().get(1).get(0), joinCond1.getChild(0));
+ Assert.assertEquals(jT.getExpressions().get(0).get(0), joinCond2.getChild(0));
+ Assert.assertEquals(jT.getExpressions().get(1).get(0), joinCond2.getChild(1));
+ }
+
+ @Test
+ public void test4WayJoin() throws SemanticException {
+ QBJoinTree jT1 = createJoinTree(JoinType.INNER, "a", null, "b");
+ QBJoinTree jT2 = createJoinTree(JoinType.INNER, "b", jT1, "c");
+ QBJoinTree jT = createJoinTree(JoinType.INNER, "c", jT2, "d");
+ ASTNode joinCond1 = applyEqPredicate(jT, "a", "x", "b", "y");
+ ASTNode joinCond2 = applyEqPredicate(jT, "b", "y", "c", "z");
+ ASTNode joinCond3 = applyEqPredicate(jT, "a", "x", "c", "z");
+ Assert.assertEquals(jT1.getExpressions().get(0).get(0), joinCond1.getChild(0));
+ Assert.assertEquals(jT1.getExpressions().get(1).get(0), joinCond1.getChild(1));
+ Assert.assertEquals(jT2.getExpressions().get(0).get(0), joinCond2.getChild(0));
+ Assert.assertEquals(jT2.getExpressions().get(1).get(0), joinCond2.getChild(1));
+ Assert.assertEquals(jT2.getExpressions().get(0).get(1), joinCond3.getChild(0));
+ Assert.assertEquals(jT2.getExpressions().get(1).get(1), joinCond3.getChild(1));
+ }
+
+ @Test
+ public void test4WayJoinSwitched() throws SemanticException {
+ QBJoinTree jT1 = createJoinTree(JoinType.INNER, "a", null, "b");
+ QBJoinTree jT2 = createJoinTree(JoinType.INNER, "b", jT1, "c");
+ QBJoinTree jT = createJoinTree(JoinType.INNER, "c", jT2, "d");
+ ASTNode joinCond1 = applyEqPredicate(jT, "b", "y", "a", "x");
+ ASTNode joinCond2 = applyEqPredicate(jT, "b", "y", "c", "z");
+ ASTNode joinCond3 = applyEqPredicate(jT, "c", "z", "a", "x");
+ Assert.assertEquals(jT1.getExpressions().get(0).get(0), joinCond1.getChild(1));
+ Assert.assertEquals(jT1.getExpressions().get(1).get(0), joinCond1.getChild(0));
+ Assert.assertEquals(jT2.getExpressions().get(0).get(0), joinCond2.getChild(0));
+ Assert.assertEquals(jT2.getExpressions().get(1).get(0), joinCond2.getChild(1));
+ Assert.assertEquals(jT2.getExpressions().get(0).get(1), joinCond3.getChild(1));
+ Assert.assertEquals(jT2.getExpressions().get(1).get(1), joinCond3.getChild(0));
+ }
+}
\ No newline at end of file
Added: hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_1.q?rev=1537500&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_1.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_1.q Thu Oct 31 14:10:54 2013
@@ -0,0 +1,30 @@
+DROP TABLE part;
+
+-- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+);
+
+LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part;
+
+
+
+explain select *
+from part p1 join part p2 join part p3 on p1.p_name = p2.p_name and p2.p_name = p3.p_name;
+
+explain select *
+from part p1 join part p2 join part p3 on p2.p_name = p1.p_name and p3.p_name = p2.p_name;
+
+explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name;
+
+explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name;
Added: hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_2.q?rev=1537500&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/join_cond_pushdown_2.q Thu Oct 31 14:10:54 2013
@@ -0,0 +1,24 @@
+DROP TABLE part;
+
+-- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+);
+
+LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part;
+
+
+explain select *
+from part p1 join part p2 join part p3 on p1.p_name = p2.p_name join part p4 on p2.p_name = p3.p_name and p1.p_name = p4.p_name;
+
+explain select *
+from part p1 join part p2 join part p3 on p2.p_name = p1.p_name join part p4 on p2.p_name = p3.p_name and p1.p_partkey = p4.p_partkey
+ and p1.p_partkey = p2.p_partkey;
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out?rev=1537500&r1=1537499&r2=1537500&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out Thu Oct 31 14:10:54 2013
@@ -280,8 +280,8 @@ STAGE PLANS:
HashTable Sink Operator
condition expressions:
0
- 1 {key}
- 2 {key}
+ 1
+ 2
handleSkewJoin: false
keys:
0 [Column[key]]
@@ -295,8 +295,8 @@ STAGE PLANS:
HashTable Sink Operator
condition expressions:
0
- 1 {key}
- 2 {key}
+ 1
+ 2
handleSkewJoin: false
keys:
0 [Column[key]]
@@ -330,44 +330,38 @@ STAGE PLANS:
Inner Join 1 to 2
condition expressions:
0
- 1 {key}
- 2 {key}
+ 1
+ 2
handleSkewJoin: false
keys:
0 [Column[key]]
1 [Column[key]]
2 [Column[key]]
- outputColumnNames: _col5, _col10
Position of Big Table: 2
- Filter Operator
- isSamplingPred: false
- predicate:
- expr: (_col10 = _col5)
- type: boolean
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- handleSkewJoin: false
- keys:
- 0 []
- 1 []
- Position of Big Table: 0
- Select Operator
- Group By Operator
- aggregations:
- expr: count()
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 []
+ 1 []
+ Position of Big Table: 0
+ Select Operator
+ Group By Operator
+ aggregations:
+ expr: count()
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
Local Work:
Map Reduce Local Work
Path -> Alias:
Added: hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out?rev=1537500&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out Thu Oct 31 14:10:54 2013
@@ -0,0 +1,919 @@
+PREHOOK: query: DROP TABLE part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: -- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- data setup
+CREATE TABLE part(
+ p_partkey INT,
+ p_name STRING,
+ p_mfgr STRING,
+ p_brand STRING,
+ p_type STRING,
+ p_size INT,
+ p_container STRING,
+ p_retailprice DOUBLE,
+ p_comment STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@part
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part
+PREHOOK: type: LOAD
+PREHOOK: Output: default@part
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/part_tiny.txt' overwrite into table part
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@part
+PREHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p1.p_name = p2.p_name and p2.p_name = p3.p_name
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p1.p_name = p2.p_name and p2.p_name = p3.p_name
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME part) p1) (TOK_TABREF (TOK_TABNAME part) p2)) (TOK_TABREF (TOK_TABNAME part) p3) (and (= (. (TOK_TABLE_OR_COL p1) p_name) (. (TOK_TABLE_OR_COL p2) p_name)) (= (. (TOK_TABLE_OR_COL p2) p_name) (. (TOK_TABLE_OR_COL p3) p_name))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ p1
+ TableScan
+ alias: p1
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 0
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p2
+ TableScan
+ alias: p2
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p3
+ TableScan
+ alias: p3
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 2
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 2 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col22
+ type: int
+ expr: _col23
+ type: string
+ expr: _col24
+ type: string
+ expr: _col25
+ type: string
+ expr: _col26
+ type: string
+ expr: _col27
+ type: int
+ expr: _col28
+ type: string
+ expr: _col29
+ type: double
+ expr: _col30
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_name = p1.p_name and p3.p_name = p2.p_name
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_name = p1.p_name and p3.p_name = p2.p_name
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME part) p1) (TOK_TABREF (TOK_TABNAME part) p2)) (TOK_TABREF (TOK_TABNAME part) p3) (and (= (. (TOK_TABLE_OR_COL p2) p_name) (. (TOK_TABLE_OR_COL p1) p_name)) (= (. (TOK_TABLE_OR_COL p3) p_name) (. (TOK_TABLE_OR_COL p2) p_name))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ p1
+ TableScan
+ alias: p1
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 0
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p2
+ TableScan
+ alias: p2
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p3
+ TableScan
+ alias: p3
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 2
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 1 to 2
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 2 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30
+ Select Operator
+ expressions:
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col22
+ type: int
+ expr: _col23
+ type: string
+ expr: _col24
+ type: string
+ expr: _col25
+ type: string
+ expr: _col26
+ type: string
+ expr: _col27
+ type: int
+ expr: _col28
+ type: string
+ expr: _col29
+ type: double
+ expr: _col30
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey + p1.p_partkey = p1.p_partkey and p3.p_name = p2.p_name
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME part) p1) (TOK_TABREF (TOK_TABNAME part) p2)) (TOK_TABREF (TOK_TABNAME part) p3) (and (= (+ (. (TOK_TABLE_OR_COL p2) p_partkey) (. (TOK_TABLE_OR_COL p1) p_partkey)) (. (TOK_TABLE_OR_COL p1) p_partkey)) (= (. (TOK_TABLE_OR_COL p3) p_name) (. (TOK_TABLE_OR_COL p2) p_name))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ p1
+ TableScan
+ alias: p1
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p2
+ TableScan
+ alias: p2
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19
+ Filter Operator
+ predicate:
+ expr: ((_col11 + _col0) = _col0)
+ type: boolean
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ TableScan
+ Reduce Output Operator
+ key expressions:
+ expr: _col12
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col12
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ p3
+ TableScan
+ alias: p3
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col19}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30
+ Select Operator
+ expressions:
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ expr: _col22
+ type: int
+ expr: _col23
+ type: string
+ expr: _col24
+ type: string
+ expr: _col25
+ type: string
+ expr: _col26
+ type: string
+ expr: _col27
+ type: int
+ expr: _col28
+ type: string
+ expr: _col29
+ type: double
+ expr: _col30
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select *
+from part p1 join part p2 join part p3 on p2.p_partkey = 1 and p3.p_name = p2.p_name
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME part) p1) (TOK_TABREF (TOK_TABNAME part) p2)) (TOK_TABREF (TOK_TABNAME part) p3) (and (= (. (TOK_TABLE_OR_COL p2) p_partkey) 1) (= (. (TOK_TABLE_OR_COL p3) p_name) (. (TOK_TABLE_OR_COL p2) p_name))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ p1
+ TableScan
+ alias: p1
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ p2
+ TableScan
+ alias: p2
+ Filter Operator
+ predicate:
+ expr: (p_partkey = 1)
+ type: boolean
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ TableScan
+ Reduce Output Operator
+ key expressions:
+ expr: _col12
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col12
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ p3
+ TableScan
+ alias: p3
+ Reduce Output Operator
+ key expressions:
+ expr: p_name
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: p_name
+ type: string
+ tag: 1
+ value expressions:
+ expr: p_partkey
+ type: int
+ expr: p_name
+ type: string
+ expr: p_mfgr
+ type: string
+ expr: p_brand
+ type: string
+ expr: p_type
+ type: string
+ expr: p_size
+ type: int
+ expr: p_container
+ type: string
+ expr: p_retailprice
+ type: double
+ expr: p_comment
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8} {VALUE._col11} {VALUE._col12} {VALUE._col13} {VALUE._col14} {VALUE._col15} {VALUE._col16} {VALUE._col17} {VALUE._col18} {VALUE._col19}
+ 1 {VALUE._col0} {VALUE._col1} {VALUE._col2} {VALUE._col3} {VALUE._col4} {VALUE._col5} {VALUE._col6} {VALUE._col7} {VALUE._col8}
+ handleSkewJoin: false
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30
+ Select Operator
+ expressions:
+ expr: _col11
+ type: int
+ expr: _col12
+ type: string
+ expr: _col13
+ type: string
+ expr: _col14
+ type: string
+ expr: _col15
+ type: string
+ expr: _col16
+ type: int
+ expr: _col17
+ type: string
+ expr: _col18
+ type: double
+ expr: _col19
+ type: string
+ expr: _col0
+ type: int
+ expr: _col1
+ type: string
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ expr: _col4
+ type: string
+ expr: _col5
+ type: int
+ expr: _col6
+ type: string
+ expr: _col7
+ type: double
+ expr: _col8
+ type: string
+ expr: _col22
+ type: int
+ expr: _col23
+ type: string
+ expr: _col24
+ type: string
+ expr: _col25
+ type: string
+ expr: _col26
+ type: string
+ expr: _col27
+ type: int
+ expr: _col28
+ type: string
+ expr: _col29
+ type: double
+ expr: _col30
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+