You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jp...@apache.org on 2015/05/07 03:25:57 UTC
hive git commit: HIVE-10506: CBO (Calcite Return Path): Disallow
return path to be enable if CBO is off (Jesus Camacho Rodriguez via Laljo
John Pullokkaran)
Repository: hive
Updated Branches:
refs/heads/master ecde4ae96 -> 93995c8be
HIVE-10506: CBO (Calcite Return Path): Disallow return path to be enable if CBO is off (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93995c8b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93995c8b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93995c8b
Branch: refs/heads/master
Commit: 93995c8be3dedc8785ced64939c608ae2433d4af
Parents: ecde4ae
Author: John Pullokkaran <jp...@hortonworks.com>
Authored: Wed May 6 18:15:33 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Wed May 6 18:21:19 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 10 +++++++++-
.../hive/ql/optimizer/IdentityProjectRemover.java | 12 ++++++++++++
.../hive/ql/optimizer/NonBlockingOpDeDupProc.java | 11 +++++++++++
.../org/apache/hadoop/hive/ql/optimizer/Optimizer.java | 8 +++-----
.../calcite/translator/HiveOpConverterPostProc.java | 10 ++++++++++
.../org/apache/hadoop/hive/ql/parse/CalcitePlanner.java | 1 +
6 files changed, 46 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 9692738..a74bbbe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -23,7 +23,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -86,6 +85,7 @@ public class Context {
protected int pathid = 10000;
protected boolean explain = false;
protected String cboInfo;
+ protected boolean cboSucceeded;
protected boolean explainLogical = false;
protected String cmd = "";
// number of previous attempts
@@ -706,4 +706,12 @@ public class Context {
this.cboInfo = cboInfo;
}
+ public boolean isCboSucceeded() {
+ return cboSucceeded;
+ }
+
+ public void setCboSucceeded(boolean cboSucceeded) {
+ this.cboSucceeded = cboSucceeded;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java
index 433699b..e3d3ce6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IdentityProjectRemover.java
@@ -26,8 +26,10 @@ import java.util.Stack;
import com.google.common.base.Predicates;
import com.google.common.collect.Iterators;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -69,6 +71,16 @@ public class IdentityProjectRemover implements Transform {
private static final Log LOG = LogFactory.getLog(IdentityProjectRemover.class);
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
+ // 0. We check the conditions to apply this transformation,
+ // if we do not meet them we bail out
+ final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED);
+ final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
+ final boolean cboSucceeded = pctx.getContext().isCboSucceeded();
+ if(cboEnabled && returnPathEnabled && cboSucceeded) {
+ return pctx;
+ }
+
+ // 1. We apply the transformation
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("R1",
"(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover());
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java
index 95c2b0b..3006a6e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/NonBlockingOpDeDupProc.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
@@ -57,6 +58,16 @@ public class NonBlockingOpDeDupProc implements Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
+ // 0. We check the conditions to apply this transformation,
+ // if we do not meet them we bail out
+ final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED);
+ final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
+ final boolean cboSucceeded = pctx.getContext().isCboSucceeded();
+ if(cboEnabled && returnPathEnabled && cboSucceeded) {
+ return pctx;
+ }
+
+ // 1. We apply the transformation
String SEL = SelectOperator.getOperatorName();
String FIL = FilterOperator.getOperatorName();
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 58f8afe..a7cf8b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -62,11 +62,9 @@ public class Optimizer {
transformations = new ArrayList<Transform>();
- // If we are translating Calcite operators into Hive operators, we need
- // additional postprocessing
- if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
- transformations.add(new HiveOpConverterPostProc());
- }
+ // Add the additional postprocessing transformations needed if
+ // we are translating Calcite operators into Hive operators.
+ transformations.add(new HiveOpConverterPostProc());
// Add the transformation that computes the lineage information.
transformations.add(new Generator());
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java
index cdd7c7e..e7c8342 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverterPostProc.java
@@ -26,6 +26,7 @@ import java.util.Stack;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -55,6 +56,15 @@ public class HiveOpConverterPostProc implements Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
+ // 0. We check the conditions to apply this transformation,
+ // if we do not meet them we bail out
+ final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED);
+ final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP);
+ final boolean cboSucceeded = pctx.getContext().isCboSucceeded();
+ if(!(cboEnabled && returnPathEnabled && cboSucceeded)) {
+ return pctx;
+ }
+
// 1. Initialize aux data structures
this.pctx = pctx;
this.aliasToOpInfo = new HashMap<String, Operator<? extends OperatorDesc>>();
http://git-wip-us.apache.org/repos/asf/hive/blob/93995c8b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 49ad6ad..48f488f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -265,6 +265,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
sinkOp = genPlan(getQB());
LOG.info("CBO Succeeded; optimized logical plan.");
this.ctx.setCboInfo("Plan optimized by CBO.");
+ this.ctx.setCboSucceeded(true);
LOG.debug(newAST.dump());
}
} catch (Exception e) {