You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2017/05/03 18:53:46 UTC
[2/2] hive git commit: HIVE-16550: Semijoin Hints should be able to
skip the optimization if needed (Deepak Jaiswal, reviewed by Jason Dere)
HIVE-16550: Semijoin Hints should be able to skip the optimization if needed (Deepak Jaiswal, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d459665
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d459665
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d459665
Branch: refs/heads/master
Commit: 5d459665b6353756606cd0c2bf7c199170f912cc
Parents: d03261c
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed May 3 11:53:17 2017 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Wed May 3 11:53:17 2017 -0700
----------------------------------------------------------------------
.../DynamicPartitionPruningOptimization.java | 21 +-
.../calcite/translator/HiveOpConverter.java | 7 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 17 +-
.../apache/hadoop/hive/ql/parse/HintParser.g | 1 +
.../hadoop/hive/ql/parse/ParseContext.java | 9 +
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 1 +
.../hadoop/hive/ql/parse/QBParseInfo.java | 9 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 133 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 1 +
.../hive/ql/plan/ExprNodeDynamicListDesc.java | 10 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java | 17 -
.../hive/ql/ppd/SyntheticJoinPredicate.java | 4 +-
.../test/queries/clientpositive/semijoin_hint.q | 45 +-
.../clientpositive/llap/semijoin_hint.q.out | 1841 +++++++++++++++++-
14 files changed, 1988 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index e1a6952..b8c0102 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -220,14 +220,23 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
}
String tableAlias = (op == null ? "" : ((TableScanOperator) op).getConf().getAlias());
- Map<String, SemiJoinHint> hints = ctx.desc.getHints();
- SemiJoinHint sjHint = (hints != null) ? hints.get(tableAlias) : null;
keyBaseAlias = ctx.generator.getOperatorId() + "_" + tableAlias + "_" + column;
- semiJoinAttempted = generateSemiJoinOperatorPlan(
- ctx, parseContext, ts, keyBaseAlias, sjHint);
- if (!semiJoinAttempted && sjHint != null) {
- throw new SemanticException("The user hint to enforce semijoin failed required conditions");
+ Map<String, SemiJoinHint> hints = parseContext.getSemiJoinHints();
+ if (hints != null) {
+ // If hints map has no entry that would imply that user enforced
+ // no runtime filtering.
+ if (hints.size() > 0) {
+ SemiJoinHint sjHint = hints.get(tableAlias);
+ semiJoinAttempted = generateSemiJoinOperatorPlan(
+ ctx, parseContext, ts, keyBaseAlias, sjHint);
+ if (!semiJoinAttempted && sjHint != null) {
+ throw new SemanticException("The user hint to enforce semijoin failed required conditions");
+ }
+ }
+ } else {
+ semiJoinAttempted = generateSemiJoinOperatorPlan(
+ ctx, parseContext, ts, keyBaseAlias, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 40c0f3b..b9b600d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -339,8 +339,6 @@ public class HiveOpConverter {
// through Hive
String[] baseSrc = new String[joinRel.getInputs().size()];
String tabAlias = getHiveDerivedTableAlias();
- Map<String, SemiJoinHint> semiJoinHints = semanticAnalyzer.parseSemiJoinHint(
- semanticAnalyzer.getQB().getParseInfo().getHints());
// 1. Convert inputs
OpAttr[] inputs = new OpAttr[joinRel.getInputs().size()];
@@ -407,7 +405,7 @@ public class HiveOpConverter {
// 6. Generate Join operator
JoinOperator joinOp = genJoin(joinRel, joinExpressions, filterExpressions, children,
- baseSrc, tabAlias, semiJoinHints);
+ baseSrc, tabAlias);
// 7. Return result
return new OpAttr(tabAlias, newVcolsInCalcite, joinOp);
@@ -879,7 +877,7 @@ public class HiveOpConverter {
private static JoinOperator genJoin(RelNode join, ExprNodeDesc[][] joinExpressions,
List<List<ExprNodeDesc>> filterExpressions, List<Operator<?>> children,
- String[] baseSrc, String tabAlias, Map<String, SemiJoinHint> semiJoinHints)
+ String[] baseSrc, String tabAlias)
throws SemanticException {
// 1. Extract join type
@@ -1006,7 +1004,6 @@ public class HiveOpConverter {
// 4. We create the join operator with its descriptor
JoinDesc desc = new JoinDesc(exprMap, outputColumnNames, noOuterJoin, joinCondns,
filters, joinExpressions, 0);
- desc.setSemiJoinHints(semiJoinHints);
desc.setReversedExprs(reversedExprs);
desc.setFilterMap(filterMap);
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 1b054a7..5d640be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -335,7 +335,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
skipCalcitePlan = true;
} else {
PreCboCtx cboCtx = (PreCboCtx) plannerCtx;
- ASTNode oldHints = getQB().getParseInfo().getHints();
+ List<ASTNode> oldHints = new ArrayList<>();
+ // Cache the hints before CBO runs and removes them.
+ // Use the hints later in top level QB.
+ getHintsFromQB(getQB(), oldHints);
// Note: for now, we don't actually pass the queryForCbo to CBO, because
// it accepts qb, not AST, and can also access all the private stuff in
@@ -364,6 +367,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
throw new SemanticException("Create view is not supported in cbo return path.");
}
sinkOp = getOptimizedHiveOPDag();
+ if (oldHints.size() > 0) {
+ LOG.debug("Propagating hints to QB: " + oldHints);
+ getQB().getParseInfo().setHintList(oldHints);
+ }
LOG.info("CBO Succeeded; optimized logical plan.");
this.ctx.setCboInfo("Plan optimized by CBO.");
this.ctx.setCboSucceeded(true);
@@ -403,13 +410,13 @@ public class CalcitePlanner extends SemanticAnalyzer {
newAST = reAnalyzeCTASAfterCbo(newAST);
}
}
- if (oldHints != null) {
+ if (oldHints.size() > 0) {
if (getQB().getParseInfo().getHints() != null) {
- LOG.warn("Hints are not null in the optimized tree; before CBO " + oldHints.dump()
- + "; after CBO " + getQB().getParseInfo().getHints().dump());
+ LOG.warn("Hints are not null in the optimized tree; "
+ + "after CBO " + getQB().getParseInfo().getHints().dump());
} else {
LOG.debug("Propagating hints to QB: " + oldHints);
- getQB().getParseInfo().setHints(oldHints);
+ getQB().getParseInfo().setHintList(oldHints);
}
}
Phase1Ctx ctx_1 = initPhase1Ctx();
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/HintParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HintParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HintParser.g
index e110fb3..ec054b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HintParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HintParser.g
@@ -83,4 +83,5 @@ hintArgName
:
Identifier
| Number
+ | KW_NONE
;
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
index 3a1f821..6de4bcd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
@@ -123,6 +123,7 @@ public class ParseContext {
private Map<ExprNodeDesc, GroupByOperator> colExprToGBMap =
new HashMap<>();
+ private Map<String, SemiJoinHint> semiJoinHints;
public ParseContext() {
}
@@ -672,4 +673,12 @@ public class ParseContext {
public Map<ExprNodeDesc, GroupByOperator> getColExprToGBMap() {
return colExprToGBMap;
}
+
+ public void setSemiJoinHints(Map<String, SemiJoinHint> hints) {
+ this.semiJoinHints = hints;
+ }
+
+ public Map<String, SemiJoinHint> getSemiJoinHints() {
+ return semiJoinHints;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 54e37f7..51aeeed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -441,6 +441,7 @@ public final class ParseUtils {
HashSet<String> aliases = new HashSet<>();
for (int i = 0; i < select.getChildCount(); ++i) {
Tree selExpr = select.getChild(i);
+ if (selExpr.getType() == HiveParser.QUERY_HINT) continue;
assert selExpr.getType() == HiveParser.TOK_SELEXPR;
assert selExpr.getChildCount() > 0;
// Examine the last child. It could be an alias.
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
index 7bf1c59..38df5de 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
@@ -44,6 +44,7 @@ public class QBParseInfo {
private String alias;
private ASTNode joinExpr;
private ASTNode hints;
+ private List<ASTNode> hintList;
private final HashMap<String, ASTNode> aliasToSrc;
/**
* insclause-0 -> TOK_TAB ASTNode
@@ -552,6 +553,14 @@ public class QBParseInfo {
hints = hint;
}
+ public void setHintList(List<ASTNode> hintList) {
+ this.hintList = hintList;
+ }
+
+ public List<ASTNode> getHintList() {
+ return hintList;
+ }
+
public ASTNode getHints() {
return hints;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index cbbb7d0..5115fc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -8126,7 +8126,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
JoinDesc desc = new JoinDesc(exprMap, outputColumnNames,
join.getNoOuterJoin(), joinCondns, filterMap, joinKeys, 0);
- desc.setSemiJoinHints(join.getSemiJoinHint());
desc.setReversedExprs(reversedExprs);
desc.setFilterMap(join.getFilterMap());
// For outer joins, add filters that apply to more than one input
@@ -8673,11 +8672,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
LOG.info("STREAMTABLE hint honored.");
parseStreamTables(joinTree, qb);
}
-
- if (qb.getParseInfo().getHints() != null) {
- // TODO: do we need this for unique join?
- joinTree.setSemiJoinHint(parseSemiJoinHint(qb.getParseInfo().getHints()));
- }
return joinTree;
}
@@ -8976,8 +8970,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
if ((conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) == false) {
parseStreamTables(joinTree, qb);
}
-
- joinTree.setSemiJoinHint(parseSemiJoinHint(qb.getParseInfo().getHints()));
}
return joinTree;
@@ -9031,46 +9023,65 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
* 2. TableName, bloom filter entries, and
* 3. TableName, ColumnName
* */
- public Map<String, SemiJoinHint> parseSemiJoinHint(ASTNode hints) throws SemanticException {
- if (hints == null) return null;
+ private Map<String, SemiJoinHint> parseSemiJoinHint(List<ASTNode> hints) throws SemanticException {
+ if (hints == null || hints.size() == 0) return null;
Map<String, SemiJoinHint> result = null;
- for (Node hintNode : hints.getChildren()) {
- ASTNode hint = (ASTNode) hintNode;
- if (hint.getChild(0).getType() != HintParser.TOK_LEFTSEMIJOIN) continue;
- if (result == null) {
- result = new HashMap<>();
- }
- String alias = null;
- String colName = null;
- Tree args = hint.getChild(1);
- for (int i = 0; i < args.getChildCount(); i++) {
- // We can have table names, column names or sizes here (or incorrect hint if the user is so inclined).
- String text = args.getChild(i).getText();
- Integer number = null;
- try {
- number = Integer.parseInt(text);
- } catch (NumberFormatException ex) { // Ignore.
+ for (ASTNode hintNode : hints) {
+ for (Node node : hintNode.getChildren()) {
+ ASTNode hint = (ASTNode) node;
+ if (hint.getChild(0).getType() != HintParser.TOK_LEFTSEMIJOIN) continue;
+ if (result == null) {
+ result = new HashMap<>();
+ }
+ String alias = null;
+ String colName = null;
+ Tree args = hint.getChild(1);
+ if (args.getChildCount() == 1) {
+ String text = args.getChild(0).getText();
+ if (text.equalsIgnoreCase("None")) {
+ // Hint to disable runtime filtering.
+ return result;
+ }
}
- if (number != null) {
- if (alias == null) {
- throw new SemanticException("Invalid semijoin hint - arg " + i + " ("
- + text + ") is a number but the previous one is not an alias");
+ for (int i = 0; i < args.getChildCount(); i++) {
+ // We can have table names, column names or sizes here (or incorrect hint if the user is so inclined).
+ String text = args.getChild(i).getText();
+ Integer number = null;
+ try {
+ number = Integer.parseInt(text);
+ } catch (NumberFormatException ex) { // Ignore.
}
- SemiJoinHint sjHint = new SemiJoinHint(alias, colName, number);
- result.put(alias, sjHint);
- alias = null;
- colName = null;
- } else {
- if (alias == null) {
- alias = text;
- } else if (colName == null ){
- colName = text;
- } else {
- // No bloom filter entries provided.
- SemiJoinHint sjHint = new SemiJoinHint(alias, colName, null);
+ if (number != null) {
+ if (alias == null) {
+ throw new SemanticException("Invalid semijoin hint - arg " + i + " ("
+ + text + ") is a number but the previous one is not an alias");
+ }
+ if (result.get(alias) != null) {
+ // A hint with same alias already present, throw
+ throw new SemanticException("A hint with alias " + alias +
+ " already present. Please use unique aliases");
+ }
+ SemiJoinHint sjHint = new SemiJoinHint(alias, colName, number);
result.put(alias, sjHint);
- alias = text;
+ alias = null;
colName = null;
+ } else {
+ if (alias == null) {
+ alias = text;
+ } else if (colName == null) {
+ colName = text;
+ } else {
+ // No bloom filter entries provided.
+ if (result.get(alias) != null) {
+ // A hint with same alias already present, throw
+ throw new SemanticException("A hint with alias " + alias +
+ " already present. Please use unique aliases");
+ }
+ SemiJoinHint sjHint = new SemiJoinHint(alias, colName, null);
+ result.put(alias, sjHint);
+ alias = text;
+ colName = null;
+ }
}
}
}
@@ -11184,7 +11195,40 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ex);
}
}
+
+ public void getHintsFromQB(QB qb, List<ASTNode> hints) {
+ if (qb.getParseInfo().getHints() != null) {
+ hints.add(qb.getParseInfo().getHints());
+ }
+
+ Set<String> aliases = qb.getSubqAliases();
+
+ for (String alias : aliases) {
+ getHintsFromQB(qb.getSubqForAlias(alias), hints);
+ }
+ }
+
+ public void getHintsFromQB(QBExpr qbExpr, List<ASTNode> hints) {
+ QBExpr qbExpr1 = qbExpr.getQBExpr1();
+ QBExpr qbExpr2 = qbExpr.getQBExpr2();
+ QB qb = qbExpr.getQB();
+
+ if (qbExpr1 != null) {
+ getHintsFromQB(qbExpr1, hints);
+ }
+ if (qbExpr2 != null) {
+ getHintsFromQB(qbExpr2, hints);
+ }
+ if (qb != null) {
+ getHintsFromQB(qb, hints);
+ }
+ }
+
Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
+ // fetch all the hints in qb
+ List<ASTNode> hintsList = new ArrayList<>();
+ getHintsFromQB(qb, hintsList);
+ getQB().getParseInfo().setHintList(hintsList);
return genPlan(qb);
}
@@ -11243,6 +11287,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
analyzeRewrite, tableDesc, createVwDesc, queryProperties, viewProjectToTableSchema, acidFileSinks);
+ // Set the semijoin hints in parse context
+ pCtx.setSemiJoinHints(parseSemiJoinHint(getQB().getParseInfo().getHintList()));
+
// 5. Take care of view creation
if (createVwDesc != null) {
if (ctx.getExplainAnalyze() == AnalyzeState.RUNNING) {
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 5ea7800..08a8f00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -533,6 +533,7 @@ public abstract class TaskCompiler {
clone.setRsToRuntimeValuesInfoMap(pCtx.getRsToRuntimeValuesInfoMap());
clone.setRsToSemiJoinBranchInfo(pCtx.getRsToSemiJoinBranchInfo());
clone.setColExprToGBMap(pCtx.getColExprToGBMap());
+ clone.setSemiJoinHints(pCtx.getSemiJoinHints());
return clone;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDynamicListDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDynamicListDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDynamicListDesc.java
index 3143554..57e27e6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDynamicListDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDynamicListDesc.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.parse.SemiJoinHint;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
/**
@@ -32,17 +31,15 @@ public class ExprNodeDynamicListDesc extends ExprNodeDesc {
Operator<? extends OperatorDesc> source;
int keyIndex;
- Map<String, SemiJoinHint> hints;
public ExprNodeDynamicListDesc() {
}
public ExprNodeDynamicListDesc(TypeInfo typeInfo, Operator<? extends OperatorDesc> source,
- int keyIndex, Map<String, SemiJoinHint> hints) {
+ int keyIndex) {
super(typeInfo);
this.source = source;
this.keyIndex = keyIndex;
- this.hints = hints;
}
public void setSource(Operator<? extends OperatorDesc> source) {
@@ -63,7 +60,7 @@ public class ExprNodeDynamicListDesc extends ExprNodeDesc {
@Override
public ExprNodeDesc clone() {
- return new ExprNodeDynamicListDesc(typeInfo, source, keyIndex, hints);
+ return new ExprNodeDynamicListDesc(typeInfo, source, keyIndex);
}
@Override
@@ -84,7 +81,4 @@ public class ExprNodeDynamicListDesc extends ExprNodeDesc {
return source.toString();
}
- public Map<String, SemiJoinHint> getHints() {
- return hints;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index 7d4267d..c4fb3f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -29,7 +29,6 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.parse.QBJoinTree;
-import org.apache.hadoop.hive.ql.parse.SemiJoinHint;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -107,10 +106,6 @@ public class JoinDesc extends AbstractOperatorDesc {
private transient Map<String, Operator<? extends OperatorDesc>> aliasToOpInfo;
private transient boolean leftInputJoin;
private transient List<String> streamAliases;
- // Note: there are two things in Hive called semi-joins - the left semi join construct,
- // and also a bloom-filter based optimization that came later. This is for the latter.
- // Everything else in this desc that says "semi-join" is for the former.
- private transient Map<String, SemiJoinHint> semiJoinHints;
// non-transient field, used at runtime to kill a task if it exceeded memory limits when running in LLAP
protected long noConditionalTaskSize;
@@ -206,7 +201,6 @@ public class JoinDesc extends AbstractOperatorDesc {
this.filterMap = clone.filterMap;
this.residualFilterExprs = clone.residualFilterExprs;
this.statistics = clone.statistics;
- this.semiJoinHints = clone.semiJoinHints;
this.noConditionalTaskSize = clone.noConditionalTaskSize;
}
@@ -694,17 +688,6 @@ public class JoinDesc extends AbstractOperatorDesc {
}
private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(JoinDesc.class);
- public void setSemiJoinHints(Map<String, SemiJoinHint> semiJoinHints) {
- if (semiJoinHints != null || this.semiJoinHints != null) {
- LOG.debug("Setting semi-join hints to " + semiJoinHints);
- }
- this.semiJoinHints = semiJoinHints;
- }
-
- public Map<String, SemiJoinHint> getSemiJoinHints() {
- return semiJoinHints;
- }
-
public long getNoConditionalTaskSize() {
return noConditionalTaskSize;
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
index f45daa8..64baa6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
@@ -26,7 +26,6 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
-import org.apache.hadoop.hive.ql.parse.SemiJoinHint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -140,7 +139,6 @@ public class SyntheticJoinPredicate extends Transform {
ReduceSinkOperator source = (ReduceSinkOperator) stack.get(stack.size() - 2);
int srcPos = join.getParentOperators().indexOf(source);
- Map<String, SemiJoinHint> hints = join.getConf().getSemiJoinHints();
List<Operator<? extends OperatorDesc>> parents = join.getParentOperators();
@@ -181,7 +179,7 @@ public class SyntheticJoinPredicate extends Transform {
inArgs.add(sourceKeys.get(i));
ExprNodeDynamicListDesc dynamicExpr =
- new ExprNodeDynamicListDesc(targetKeys.get(i).getTypeInfo(), target, i, hints);
+ new ExprNodeDynamicListDesc(targetKeys.get(i).getTypeInfo(), target, i);
inArgs.add(dynamicExpr);
http://git-wip-us.apache.org/repos/asf/hive/blob/5d459665/ql/src/test/queries/clientpositive/semijoin_hint.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/semijoin_hint.q b/ql/src/test/queries/clientpositive/semijoin_hint.q
index 5de0c8c..a3cd1d6 100644
--- a/ql/src/test/queries/clientpositive/semijoin_hint.q
+++ b/ql/src/test/queries/clientpositive/semijoin_hint.q
@@ -35,20 +35,55 @@ analyze table alltypesorc_int compute statistics for columns;
analyze table srcpart_date compute statistics for columns;
analyze table srcpart_small compute statistics for columns;
+create table srccc as select * from src;
+
set hive.cbo.returnpath.hiveop=true;
-create table srccc as select * from src;
+-- disabling this test case for returnpath true as the aliases in case of union are mangled due to which hints are not excercised.
+--explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small s on (k.str = s.key1)
+-- union all
+-- select /*+ semi(v, 5000)*/ count(*) from srcpart_date d join srcpart_small v on (d.str = v.key1);
+
+-- Query which creates semijoin
+explain select count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+-- Skip semijoin by using keyword "None" as argument
+explain select /*+ semi(None)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
-EXPLAIN select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1) join alltypesorc_int i on (k.value = i.cstring);
+EXPLAIN select /*+ semi(srcpart_date, str, 5000)*/ count(*) from srcpart_date join srcpart_small v on (srcpart_date.str = v.key1) join alltypesorc_int i on (srcpart_date.value = i.cstring);
EXPLAIN select /*+ semi(i, 3000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1) join alltypesorc_int i on (v.key1 = i.cstring);
-explain select /*+ semi(k, str, 1000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
set hive.cbo.returnpath.hiveop=false;
-explain select /*+ semi(k, 1000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small s on (k.str = s.key1)
+ union all
+ select /*+ semi(v, 5000)*/ count(*) from srcpart_date d join srcpart_small v on (d.str = v.key1);
+
+-- Query which creates semijoin
+explain select count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+-- Skip semijoin by using keyword "None" as argument
+explain select /*+ semi(None)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+
+EXPLAIN select /*+ semi(srcpart_date, str, 5000)*/ count(*) from srcpart_date join srcpart_small v on (srcpart_date.str = v.key1) join alltypesorc_int i on (srcpart_date.value = i.cstring);
+EXPLAIN select /*+ semi(i, 3000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1) join alltypesorc_int i on (v.key1 = i.cstring);
+
+explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
set hive.cbo.enable=false;
-explain select /*+ semi(k, str, 1000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small s on (k.str = s.key1)
+ union all
+ select /*+ semi(v, 5000)*/ count(*) from srcpart_date d join srcpart_small v on (d.str = v.key1);
+
+-- Query which creates semijoin
+explain select count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+-- Skip semijoin by using keyword "None" as argument
+explain select /*+ semi(None)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+
+EXPLAIN select /*+ semi(srcpart_date, str, 5000)*/ count(*) from srcpart_date join srcpart_small v on (srcpart_date.str = v.key1) join alltypesorc_int i on (srcpart_date.value = i.cstring);
+EXPLAIN select /*+ semi(i, 3000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1) join alltypesorc_int i on (v.key1 = i.cstring);
+
+explain select /*+ semi(k, str, 5000)*/ count(*) from srcpart_date k join srcpart_small v on (k.str = v.key1);
+