You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/18 00:00:46 UTC
[16/50] [abbrv] hive git commit: HIVE-11416: CBO: Calcite Operator To
Hive Operator (Calcite Return Path): Groupby Optimizer assumes the schema can
match after removing RS and GBY (reviewed by Jesus Camacho Rodriguez)
HIVE-11416: CBO: Calcite Operator To Hive Operator (Calcite Return Path): Groupby Optimizer assumes the schema can match after removing RS and GBY (reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/763cb02b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/763cb02b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/763cb02b
Branch: refs/heads/hbase-metastore
Commit: 763cb02b5eafb0ecd3fd0eb512636a1b092df671
Parents: 57ba795
Author: Pengcheng Xiong <px...@apache.org>
Authored: Tue Aug 11 11:26:48 2015 -0700
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Tue Aug 11 11:26:48 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/Operator.java | 25 ---------
.../hive/ql/optimizer/GroupByOptimizer.java | 58 +++++++++++++++++++-
2 files changed, 57 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/763cb02b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 0f02737..acbe504 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -769,31 +769,6 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
}
}
- // Remove the operators till a certain depth.
- // Return true if the remove was successful, false otherwise
- public boolean removeChildren(int depth) {
- Operator<? extends OperatorDesc> currOp = this;
- for (int i = 0; i < depth; i++) {
- // If there are more than 1 children at any level, don't do anything
- if ((currOp.getChildOperators() == null) || (currOp.getChildOperators().isEmpty()) ||
- (currOp.getChildOperators().size() > 1)) {
- return false;
- }
- currOp = currOp.getChildOperators().get(0);
- }
-
- setChildOperators(currOp.getChildOperators());
-
- List<Operator<? extends OperatorDesc>> parentOps =
- new ArrayList<Operator<? extends OperatorDesc>>();
- parentOps.add(this);
-
- for (Operator<? extends OperatorDesc> op : currOp.getChildOperators()) {
- op.setParentOperators(parentOps);
- }
- return true;
- }
-
/**
* Replace one parent with another at the same position. Chilren of the new
* parent are not updated
http://git-wip-us.apache.org/repos/asf/hive/blob/763cb02b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
index af54286..ce3f59a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -31,9 +32,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -520,12 +524,64 @@ public class GroupByOptimizer implements Transform {
return;
}
- if (groupByOp.removeChildren(depth)) {
+ if (removeChildren(groupByOp, depth)) {
// Use bucketized hive input format - that makes sure that one mapper reads the entire file
groupByOp.setUseBucketizedHiveInputFormat(true);
groupByOp.getConf().setMode(GroupByDesc.Mode.FINAL);
}
}
+
+ // Remove the operators till a certain depth.
+ // Return true if the remove was successful, false otherwise
+ public boolean removeChildren(Operator<? extends OperatorDesc> currOp, int depth) {
+ Operator<? extends OperatorDesc> inputOp = currOp;
+ for (int i = 0; i < depth; i++) {
+ // If there are more than 1 children at any level, don't do anything
+ if ((currOp.getChildOperators() == null) || (currOp.getChildOperators().isEmpty())
+ || (currOp.getChildOperators().size() > 1)) {
+ return false;
+ }
+ currOp = currOp.getChildOperators().get(0);
+ }
+
+ // add selectOp to match the schema
+ // after that, inputOp is the parent of selOp.
+ for (Operator<? extends OperatorDesc> op : inputOp.getChildOperators()) {
+ op.getParentOperators().clear();
+ }
+ inputOp.getChildOperators().clear();
+ Operator<? extends OperatorDesc> selOp = genOutputSelectForGroupBy(inputOp, currOp);
+
+ // update the childOp of selectOp
+ selOp.setChildOperators(currOp.getChildOperators());
+
+ // update the parentOp
+ for (Operator<? extends OperatorDesc> op : currOp.getChildOperators()) {
+ op.replaceParent(currOp, selOp);
+ }
+ return true;
+ }
+
+ private Operator<? extends OperatorDesc> genOutputSelectForGroupBy(
+ Operator<? extends OperatorDesc> parentOp, Operator<? extends OperatorDesc> currOp) {
+ assert (parentOp.getSchema().getSignature().size() == currOp.getSchema().getSignature().size());
+ Iterator<ColumnInfo> pIter = parentOp.getSchema().getSignature().iterator();
+ Iterator<ColumnInfo> cIter = currOp.getSchema().getSignature().iterator();
+ List<ExprNodeDesc> columns = new ArrayList<ExprNodeDesc>();
+ List<String> colName = new ArrayList<String>();
+ Map<String, ExprNodeDesc> columnExprMap = new HashMap<String, ExprNodeDesc>();
+ while (pIter.hasNext()) {
+ ColumnInfo pInfo = pIter.next();
+ ColumnInfo cInfo = cIter.next();
+ ExprNodeDesc column = new ExprNodeColumnDesc(pInfo.getType(), pInfo.getInternalName(),
+ pInfo.getTabAlias(), pInfo.getIsVirtualCol(), pInfo.isSkewedCol());
+ columns.add(column);
+ colName.add(cInfo.getInternalName());
+ columnExprMap.put(cInfo.getInternalName(), column);
+ }
+ return OperatorFactory.getAndMakeChild(new SelectDesc(columns, colName), new RowSchema(currOp
+ .getSchema().getSignature()), columnExprMap, parentOp);
+ }
}
/**