You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/25 23:28:57 UTC
[27/50] [abbrv] hive git commit: HIVE-11569 : Use PreOrderOnceWalker
where feasible (Ashutosh Chauhan via Jesus Camacho Rodriguez)
HIVE-11569 : Use PreOrderOnceWalker where feasible (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e1b88a19
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e1b88a19
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e1b88a19
Branch: refs/heads/llap
Commit: e1b88a19a98534cb7c2f6d46e39f5eb135ab6b50
Parents: 6433c3b
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Tue Aug 18 09:13:59 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Aug 18 09:13:59 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/index/RewriteCanApplyCtx.java | 8 ++++----
.../optimizer/physical/NullScanTaskDispatcher.java | 6 +++---
.../hive/ql/optimizer/physical/Vectorizer.java | 16 +++-------------
.../hadoop/hive/ql/ppd/SyntheticJoinPredicate.java | 4 ++--
4 files changed, 12 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
index b56b608..988bb29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteCanApplyCtx.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -75,7 +75,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
private String baseTableName;
private String indexTableName;
private String aggFunction;
-
+
private TableScanOperator tableScanOperator;
private List<SelectOperator> selectOperators;
private List<GroupByOperator> groupByOperators;
@@ -156,7 +156,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
/**
* This method walks all the nodes starting from topOp TableScanOperator node
* and invokes methods from {@link RewriteCanApplyProcFactory} for each of the rules
- * added to the opRules map. We use the {@link PreOrderWalker} for a pre-order
+ * added to the opRules map. We use the {@link PreOrderOnceWalker} for a pre-order
* traversal of the operator tree.
*
* The methods from {@link RewriteCanApplyProcFactory} set appropriate values in
@@ -180,7 +180,7 @@ public final class RewriteCanApplyCtx implements NodeProcessorCtx {
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, this);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of topop nodes
List<Node> topNodes = new ArrayList<Node>();
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
index f901812..8dcd283 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/NullScanTaskDispatcher.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer.WalkerCtx;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -61,7 +61,7 @@ import org.apache.hadoop.hive.serde2.NullStructSerDe;
public class NullScanTaskDispatcher implements Dispatcher {
static final Log LOG = LogFactory.getLog(NullScanTaskDispatcher.class.getName());
-
+
private final PhysicalContext physicalContext;
private final Map<Rule, NodeProcessor> rules;
@@ -177,7 +177,7 @@ public class NullScanTaskDispatcher implements Dispatcher {
// The dispatcher fires the processor corresponding to the closest
// matching rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(null, rules, walkerCtx);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of topOp nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 7ecd50a..2e3bd76 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
-import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyLongOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyMultiKeyOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinInnerBigOnlyStringOperator;
@@ -68,6 +67,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
@@ -155,7 +155,6 @@ public class Vectorizer implements PhysicalPlanResolver {
Set<String> supportedAggregationUdfs = new HashSet<String>();
- private PhysicalContext physicalContext = null;
private HiveConf hiveConf;
public Vectorizer() {
@@ -307,13 +306,10 @@ public class Vectorizer implements PhysicalPlanResolver {
class VectorizationDispatcher implements Dispatcher {
- private final PhysicalContext physicalContext;
-
private List<String> reduceColumnNames;
private List<TypeInfo> reduceTypeInfos;
public VectorizationDispatcher(PhysicalContext physicalContext) {
- this.physicalContext = physicalContext;
reduceColumnNames = null;
reduceTypeInfos = null;
}
@@ -428,7 +424,7 @@ public class Vectorizer implements PhysicalPlanResolver {
MapWorkVectorizationNodeProcessor vnp = new MapWorkVectorizationNodeProcessor(mapWork, isTez);
addMapWorkRules(opRules, vnp);
Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// iterator the mapper operator tree
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(mapWork.getAliasToWork().values());
@@ -701,12 +697,10 @@ public class Vectorizer implements PhysicalPlanResolver {
class MapWorkVectorizationNodeProcessor extends VectorizationNodeProcessor {
- private final MapWork mWork;
private final boolean isTez;
public MapWorkVectorizationNodeProcessor(MapWork mWork, boolean isTez) {
super();
- this.mWork = mWork;
this.isTez = isTez;
}
@@ -765,7 +759,7 @@ public class Vectorizer implements PhysicalPlanResolver {
private final List<String> reduceColumnNames;
private final List<TypeInfo> reduceTypeInfos;
- private boolean isTez;
+ private final boolean isTez;
private Operator<? extends OperatorDesc> rootVectorOp;
@@ -865,7 +859,6 @@ public class Vectorizer implements PhysicalPlanResolver {
@Override
public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
- this.physicalContext = physicalContext;
hiveConf = physicalContext.getConf();
boolean vectorPath = HiveConf.getBoolVar(hiveConf,
@@ -1406,7 +1399,6 @@ public class Vectorizer implements PhysicalPlanResolver {
int[] smallTableIndices;
int smallTableIndicesSize;
- List<ExprNodeDesc> smallTableExprs = desc.getExprs().get(posSingleVectorMapJoinSmallTable);
if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) {
smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable);
LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices " + Arrays.toString(smallTableIndices));
@@ -1445,8 +1437,6 @@ public class Vectorizer implements PhysicalPlanResolver {
Operator<? extends OperatorDesc> vectorOp = null;
Class<? extends Operator<?>> opClass = null;
- boolean isOuterJoin = !desc.getNoOuterJoin();
-
VectorMapJoinDesc.HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE;
VectorMapJoinDesc.HashTableKind hashTableKind = HashTableKind.NONE;
VectorMapJoinDesc.HashTableKeyType hashTableKeyType = HashTableKeyType.NONE;
http://git-wip-us.apache.org/repos/asf/hive/blob/e1b88a19/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
index 32af813..44159c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.PreOrderWalker;
+import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.optimizer.Transform;
@@ -92,7 +92,7 @@ public class SyntheticJoinPredicate implements Transform {
// rule and passes the context along
SyntheticContext context = new SyntheticContext(pctx);
Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context);
- GraphWalker ogw = new PreOrderWalker(disp);
+ GraphWalker ogw = new PreOrderOnceWalker(disp);
// Create a list of top op nodes
List<Node> topNodes = new ArrayList<Node>();