You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/04/28 20:27:30 UTC
hive git commit: HIVE-10416 : CBO (Calcite Return Path): Fix return
columns if Sort operator is on top of plan returned by Calcite (Jesus Camacho
Rodriguez via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master c90cb3ad6 -> 35513683a
HIVE-10416 : CBO (Calcite Return Path): Fix return columns if Sort operator is on top of plan returned by Calcite (Jesus Camacho Rodriguez via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/35513683
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/35513683
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/35513683
Branch: refs/heads/master
Commit: 35513683a7e1f4365ab3d7d001b212b3ad0bd6f4
Parents: c90cb3a
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Apr 23 18:40:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Apr 28 11:25:17 2015 -0700
----------------------------------------------------------------------
.../calcite/translator/HiveOpConverter.java | 115 +++++++++++++----
.../translator/PlanModifierForASTConv.java | 88 +------------
.../translator/PlanModifierForReturnPath.java | 41 ++++++
.../calcite/translator/PlanModifierUtil.java | 125 +++++++++++++++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 4 +-
5 files changed, 261 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/35513683/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 7311b71..85d1663 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelDistribution;
import org.apache.calcite.rel.RelDistribution.Type;
@@ -38,6 +37,7 @@ import org.apache.calcite.rel.core.SortExchange;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -367,6 +367,7 @@ public class HiveOpConverter {
Operator<?> inputOp = inputOpAf.inputs.get(0);
Operator<?> resultOp = inputOpAf.inputs.get(0);
+
// 1. If we need to sort tuples based on the value of some
// of their columns
if (sortRel.getCollation() != RelCollations.EMPTY) {
@@ -379,30 +380,51 @@ public class HiveOpConverter {
// 1.a. Extract order for each column from collation
// Generate sortCols and order
+ ImmutableBitSet.Builder sortColsPosBuilder = new ImmutableBitSet.Builder();
+ ImmutableBitSet.Builder sortOutputColsPosBuilder = new ImmutableBitSet.Builder();
+ Map<Integer, RexNode> obRefToCallMap = sortRel.getInputRefToCallMap();
List<ExprNodeDesc> sortCols = new ArrayList<ExprNodeDesc>();
StringBuilder order = new StringBuilder();
- for (RelCollation collation : sortRel.getCollationList()) {
- for (RelFieldCollation sortInfo : collation.getFieldCollations()) {
- int sortColumnPos = sortInfo.getFieldIndex();
- ColumnInfo columnInfo = new ColumnInfo(inputOp.getSchema().getSignature()
- .get(sortColumnPos));
- ExprNodeColumnDesc sortColumn = new ExprNodeColumnDesc(columnInfo.getType(),
- columnInfo.getInternalName(), columnInfo.getTabAlias(), columnInfo.getIsVirtualCol());
- sortCols.add(sortColumn);
- if (sortInfo.getDirection() == RelFieldCollation.Direction.DESCENDING) {
- order.append("-");
- } else {
- order.append("+");
+ for (RelFieldCollation sortInfo : sortRel.getCollation().getFieldCollations()) {
+ int sortColumnPos = sortInfo.getFieldIndex();
+ ColumnInfo columnInfo = new ColumnInfo(inputOp.getSchema().getSignature()
+ .get(sortColumnPos));
+ ExprNodeColumnDesc sortColumn = new ExprNodeColumnDesc(columnInfo.getType(),
+ columnInfo.getInternalName(), columnInfo.getTabAlias(), columnInfo.getIsVirtualCol());
+ sortCols.add(sortColumn);
+ if (sortInfo.getDirection() == RelFieldCollation.Direction.DESCENDING) {
+ order.append("-");
+ } else {
+ order.append("+");
+ }
+
+ if (obRefToCallMap != null) {
+ RexNode obExpr = obRefToCallMap.get(sortColumnPos);
+ sortColsPosBuilder.set(sortColumnPos);
+ if (obExpr == null) {
+ sortOutputColsPosBuilder.set(sortColumnPos);
}
}
}
// Use only 1 reducer for order by
int numReducers = 1;
+ // We keep the columns only the columns that are part of the final output
+ List<String> keepColumns = new ArrayList<String>();
+ final ImmutableBitSet sortColsPos = sortColsPosBuilder.build();
+ final ImmutableBitSet sortOutputColsPos = sortOutputColsPosBuilder.build();
+ final ArrayList<ColumnInfo> inputSchema = inputOp.getSchema().getSignature();
+ for (int pos=0; pos<inputSchema.size(); pos++) {
+ if ((sortColsPos.get(pos) && sortOutputColsPos.get(pos)) ||
+ (!sortColsPos.get(pos) && !sortOutputColsPos.get(pos))) {
+ keepColumns.add(inputSchema.get(pos).getInternalName());
+ }
+ }
+
// 1.b. Generate reduce sink and project operator
resultOp = genReduceSinkAndBacktrackSelect(resultOp,
sortCols.toArray(new ExprNodeDesc[sortCols.size()]), 0, new ArrayList<ExprNodeDesc>(),
- order.toString(), numReducers, Operation.NOT_ACID, strictMode);
+ order.toString(), numReducers, Operation.NOT_ACID, strictMode, keepColumns);
}
// 2. If we need to generate limit
@@ -587,18 +609,27 @@ public class HiveOpConverter {
}
private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input,
+ ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order,
+ int numReducers, Operation acidOperation, boolean strictMode) throws SemanticException {
+ return genReduceSinkAndBacktrackSelect(input, keys, tag, partitionCols, order,
+ numReducers, acidOperation, strictMode, input.getSchema().getColumnNames());
+ }
+
+ private static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input,
ExprNodeDesc[] keys, int tag, ArrayList<ExprNodeDesc> partitionCols, String order,
- int numReducers, Operation acidOperation, boolean strictMode) throws SemanticException {
+ int numReducers, Operation acidOperation, boolean strictMode,
+ List<String> keepColNames) throws SemanticException {
// 1. Generate RS operator
ReduceSinkOperator rsOp = genReduceSink(input, keys, tag, partitionCols, order, numReducers,
acidOperation, strictMode);
// 2. Generate backtrack Select operator
- Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSink(rsOp,
- input);
+ Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSink(keepColNames,
+ rsOp.getConf().getOutputKeyColumnNames(), rsOp.getConf().getOutputValueColumnNames(),
+ rsOp.getValueIndex(), input);
SelectDesc selectDesc = new SelectDesc(new ArrayList<ExprNodeDesc>(descriptors.values()),
new ArrayList<String>(descriptors.keySet()));
- ArrayList<ColumnInfo> cinfoLst = createColInfos(input);
+ ArrayList<ColumnInfo> cinfoLst = createColInfosSubset(input, keepColNames);
SelectOperator selectOp = (SelectOperator) OperatorFactory.getAndMakeChild(selectDesc,
new RowSchema(cinfoLst), rsOp);
selectOp.setColumnExprMap(descriptors);
@@ -766,7 +797,7 @@ public class HiveOpConverter {
posToAliasMap.put(pos, new HashSet<String>(inputRS.getSchema().getTableNames()));
- Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSink(outputPos,
+ Map<String, ExprNodeDesc> descriptors = buildBacktrackFromReduceSinkForJoin(outputPos,
outputColumnNames, keyColNames, valColNames, index, parent);
List<ColumnInfo> parentColumns = parent.getSchema().getSignature();
@@ -830,14 +861,7 @@ public class HiveOpConverter {
return resultJoinType;
}
- private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(ReduceSinkOperator rsOp,
- Operator<?> inputOp) {
- return buildBacktrackFromReduceSink(0, inputOp.getSchema().getColumnNames(), rsOp.getConf()
- .getOutputKeyColumnNames(), rsOp.getConf().getOutputValueColumnNames(),
- rsOp.getValueIndex(), inputOp);
- }
-
- private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(int initialPos,
+ private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSinkForJoin(int initialPos,
List<String> outputColumnNames, List<String> keyColNames, List<String> valueColNames,
int[] index, Operator<?> inputOp) {
Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>();
@@ -856,6 +880,29 @@ public class HiveOpConverter {
return columnDescriptors;
}
+ private static Map<String, ExprNodeDesc> buildBacktrackFromReduceSink(List<String> keepColNames,
+ List<String> keyColNames, List<String> valueColNames, int[] index, Operator<?> inputOp) {
+ Map<String, ExprNodeDesc> columnDescriptors = new LinkedHashMap<String, ExprNodeDesc>();
+ int pos = 0;
+ for (int i = 0; i < index.length; i++) {
+ ColumnInfo info = inputOp.getSchema().getSignature().get(i);
+ if (pos < keepColNames.size() &&
+ info.getInternalName().equals(keepColNames.get(pos))) {
+ String field;
+ if (index[i] >= 0) {
+ field = Utilities.ReduceField.KEY + "." + keyColNames.get(index[i]);
+ } else {
+ field = Utilities.ReduceField.VALUE + "." + valueColNames.get(-index[i] - 1);
+ }
+ ExprNodeColumnDesc desc = new ExprNodeColumnDesc(info.getType(), field, info.getTabAlias(),
+ info.getIsVirtualCol());
+ columnDescriptors.put(keepColNames.get(pos), desc);
+ pos++;
+ }
+ }
+ return columnDescriptors;
+ }
+
private static ExprNodeDesc convertToExprNode(RexNode rn, RelNode inputRel, String tabAlias, OpAttr inputAttr) {
return rn.accept(new ExprNodeConverter(tabAlias, inputRel.getRowType(), inputAttr.vcolsInCalcite,
inputRel.getCluster().getTypeFactory()));
@@ -869,6 +916,20 @@ public class HiveOpConverter {
return cInfoLst;
}
+ private static ArrayList<ColumnInfo> createColInfosSubset(Operator<?> input,
+ List<String> keepColNames) {
+ ArrayList<ColumnInfo> cInfoLst = new ArrayList<ColumnInfo>();
+ int pos = 0;
+ for (ColumnInfo ci : input.getSchema().getSignature()) {
+ if (pos < keepColNames.size() &&
+ ci.getInternalName().equals(keepColNames.get(pos))) {
+ cInfoLst.add(new ColumnInfo(ci));
+ pos++;
+ }
+ }
+ return cInfoLst;
+ }
+
private static Pair<ArrayList<ColumnInfo>, Set<Integer>> createColInfos(
List<RexNode> calciteExprs, List<ExprNodeDesc> hiveExprs, List<String> projNames,
OpAttr inpOpAf) {
http://git-wip-us.apache.org/repos/asf/hive/blob/35513683/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
index cba37bc..d8be7ff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
@@ -18,14 +18,11 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.hep.HepRelVertex;
import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelCollationImpl;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.SingleRel;
import org.apache.calcite.rel.core.Aggregate;
@@ -38,19 +35,14 @@ import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.rules.MultiJoin;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.util.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
@@ -58,11 +50,12 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
public class PlanModifierForASTConv {
+
private static final Log LOG = LogFactory.getLog(PlanModifierForASTConv.class);
+
public static RelNode convertOpTree(RelNode rel, List<FieldSchema> resultSchema)
throws CalciteSemanticException {
RelNode newTopNode = rel;
@@ -84,7 +77,7 @@ public class PlanModifierForASTConv {
}
Pair<RelNode, RelNode> topSelparentPair = HiveCalciteUtil.getTopLevelSelect(newTopNode);
- fixTopOBSchema(newTopNode, topSelparentPair, resultSchema);
+ PlanModifierUtil.fixTopOBSchema(newTopNode, topSelparentPair, resultSchema, true);
if (LOG.isDebugEnabled()) {
LOG.debug("Plan after fixTopOBSchema\n " + RelOptUtil.toString(newTopNode));
}
@@ -176,79 +169,6 @@ public class PlanModifierForASTConv {
}
}
- private static void fixTopOBSchema(final RelNode rootRel,
- Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema)
- throws CalciteSemanticException {
- if (!(topSelparentPair.getKey() instanceof Sort)
- || !HiveCalciteUtil.orderRelNode(topSelparentPair.getKey())) {
- return;
- }
- HiveSort obRel = (HiveSort) topSelparentPair.getKey();
- Project obChild = (Project) topSelparentPair.getValue();
- if (obChild.getRowType().getFieldCount() <= resultSchema.size()) {
- return;
- }
-
- RelDataType rt = obChild.getRowType();
- @SuppressWarnings({ "unchecked", "rawtypes" })
- Set<Integer> collationInputRefs = new HashSet(
- RelCollationImpl.ordinals(obRel.getCollation()));
- ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
- for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
- if (collationInputRefs.contains(i)) {
- RexNode obyExpr = obChild.getChildExps().get(i);
- if (obyExpr instanceof RexCall) {
- int a = -1;
- List<RexNode> operands = new ArrayList<>();
- for (int k = 0; k< ((RexCall) obyExpr).operands.size(); k++) {
- RexNode rn = ((RexCall) obyExpr).operands.get(k);
- for (int j = 0; j < resultSchema.size(); j++) {
- if( obChild.getChildExps().get(j).toString().equals(rn.toString())) {
- a = j;
- break;
- }
- } if (a != -1) {
- operands.add(new RexInputRef(a, rn.getType()));
- } else {
- operands.add(rn);
- }
- a = -1;
- }
- obyExpr = obChild.getCluster().getRexBuilder().makeCall(((RexCall)obyExpr).getOperator(), operands);
- }
- inputRefToCallMapBldr.put(i, obyExpr);
- }
- }
- ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
-
- if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) {
- LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size()));
- throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
- }
- // This removes order-by only expressions from the projections.
- HiveProject replacementProjectRel = HiveProject.create(obChild.getInput(), obChild
- .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames()
- .subList(0, resultSchema.size()));
- obRel.replaceInput(0, replacementProjectRel);
- obRel.setInputRefToCallMap(inputRefToCallMap);
- }
-
- private static String generateInvalidSchemaMessage(Project topLevelProj,
- List<FieldSchema> resultSchema, int fieldsForOB) {
- String errorDesc = "Result Schema didn't match Calcite Optimized Op Tree; schema: ";
- for (FieldSchema fs : resultSchema) {
- errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], ";
- }
- errorDesc += " projection fields: ";
- for (RexNode exp : topLevelProj.getChildExps()) {
- errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], ";
- }
- if (fieldsForOB != 0) {
- errorDesc += fieldsForOB + " fields removed due to ORDER BY ";
- }
- return errorDesc.substring(0, errorDesc.length() - 2);
- }
-
private static RelNode renameTopLevelSelectInResultSchema(final RelNode rootRel,
Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema)
throws CalciteSemanticException {
@@ -260,7 +180,7 @@ public class PlanModifierForASTConv {
List<RexNode> rootChildExps = originalProjRel.getChildExps();
if (resultSchema.size() != rootChildExps.size()) {
// Safeguard against potential issues in CBO RowResolver construction. Disable CBO for now.
- LOG.error(generateInvalidSchemaMessage(originalProjRel, resultSchema, 0));
+ LOG.error(PlanModifierUtil.generateInvalidSchemaMessage(originalProjRel, resultSchema, 0));
throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/35513683/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForReturnPath.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForReturnPath.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForReturnPath.java
new file mode 100644
index 0000000..06cf69d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForReturnPath.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.util.List;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.util.Pair;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+
+public class PlanModifierForReturnPath {
+
+
+ public static RelNode convertOpTree(RelNode rel, List<FieldSchema> resultSchema)
+ throws CalciteSemanticException {
+ RelNode newTopNode = rel;
+
+ Pair<RelNode, RelNode> topSelparentPair = HiveCalciteUtil.getTopLevelSelect(newTopNode);
+ PlanModifierUtil.fixTopOBSchema(newTopNode, topSelparentPair, resultSchema, false);
+
+ return newTopNode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/35513683/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
new file mode 100644
index 0000000..2c820f0
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.Pair;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
+
+import com.google.common.collect.ImmutableMap;
+
+public class PlanModifierUtil {
+
+ private static final Log LOG = LogFactory.getLog(PlanModifierUtil.class);
+
+
+ protected static void fixTopOBSchema(final RelNode rootRel,
+ Pair<RelNode, RelNode> topSelparentPair, List<FieldSchema> resultSchema,
+ boolean replaceProject) throws CalciteSemanticException {
+ if (!(topSelparentPair.getKey() instanceof Sort)
+ || !HiveCalciteUtil.orderRelNode(topSelparentPair.getKey())) {
+ return;
+ }
+ HiveSort obRel = (HiveSort) topSelparentPair.getKey();
+ Project obChild = (Project) topSelparentPair.getValue();
+ if (obChild.getRowType().getFieldCount() <= resultSchema.size()) {
+ return;
+ }
+
+ RelDataType rt = obChild.getRowType();
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ Set<Integer> collationInputRefs = new HashSet(
+ RelCollations.ordinals(obRel.getCollation()));
+ ImmutableMap.Builder<Integer, RexNode> inputRefToCallMapBldr = ImmutableMap.builder();
+ for (int i = resultSchema.size(); i < rt.getFieldCount(); i++) {
+ if (collationInputRefs.contains(i)) {
+ RexNode obyExpr = obChild.getChildExps().get(i);
+ if (obyExpr instanceof RexCall) {
+ int a = -1;
+ List<RexNode> operands = new ArrayList<>();
+ for (int k = 0; k< ((RexCall) obyExpr).operands.size(); k++) {
+ RexNode rn = ((RexCall) obyExpr).operands.get(k);
+ for (int j = 0; j < resultSchema.size(); j++) {
+ if( obChild.getChildExps().get(j).toString().equals(rn.toString())) {
+ a = j;
+ break;
+ }
+ } if (a != -1) {
+ operands.add(new RexInputRef(a, rn.getType()));
+ } else {
+ operands.add(rn);
+ }
+ a = -1;
+ }
+ obyExpr = obChild.getCluster().getRexBuilder().makeCall(((RexCall)obyExpr).getOperator(), operands);
+ }
+ inputRefToCallMapBldr.put(i, obyExpr);
+ }
+ }
+ ImmutableMap<Integer, RexNode> inputRefToCallMap = inputRefToCallMapBldr.build();
+
+ if ((obChild.getRowType().getFieldCount() - inputRefToCallMap.size()) != resultSchema.size()) {
+ LOG.error(generateInvalidSchemaMessage(obChild, resultSchema, inputRefToCallMap.size()));
+ throw new CalciteSemanticException("Result Schema didn't match Optimized Op Tree Schema");
+ }
+
+ if (replaceProject) {
+ // This removes order-by only expressions from the projections.
+ HiveProject replacementProjectRel = HiveProject.create(obChild.getInput(), obChild
+ .getChildExps().subList(0, resultSchema.size()), obChild.getRowType().getFieldNames()
+ .subList(0, resultSchema.size()));
+ obRel.replaceInput(0, replacementProjectRel);
+ }
+ obRel.setInputRefToCallMap(inputRefToCallMap);
+ }
+
+ protected static String generateInvalidSchemaMessage(Project topLevelProj,
+ List<FieldSchema> resultSchema, int fieldsForOB) {
+ String errorDesc = "Result Schema didn't match Calcite Optimized Op Tree; schema: ";
+ for (FieldSchema fs : resultSchema) {
+ errorDesc += "[" + fs.getName() + ":" + fs.getType() + "], ";
+ }
+ errorDesc += " projection fields: ";
+ for (RexNode exp : topLevelProj.getChildExps()) {
+ errorDesc += "[" + exp.toString() + ":" + exp.getType() + "], ";
+ }
+ if (fieldsForOB != 0) {
+ errorDesc += fieldsForOB + " fields removed due to ORDER BY ";
+ }
+ return errorDesc.substring(0, errorDesc.length() - 2);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/35513683/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 6a15bf6..5855695 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -147,6 +147,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinCondTypeCheckProcFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinTypeCheckCtx;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.PlanModifierForReturnPath;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
@@ -629,7 +630,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
throw new AssertionError("rethrowCalciteException didn't throw for " + e.getMessage());
}
- RelNode modifiedOptimizedOptiqPlan = introduceProjectIfNeeded(optimizedOptiqPlan);
+ RelNode modifiedOptimizedOptiqPlan = PlanModifierForReturnPath.convertOpTree(
+ introduceProjectIfNeeded(optimizedOptiqPlan), topLevelFieldSchema);
LOG.debug("Translating the following plan:\n" + RelOptUtil.toString(modifiedOptimizedOptiqPlan));
Operator<?> hiveRoot = new HiveOpConverter(this, conf, unparseTranslator, topOps,