You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/04/02 20:50:06 UTC
hive git commit: HIVE-19073: StatsOptimizer may mangle constant
columns (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 3660ac24b -> 2d770d8de
HIVE-19073: StatsOptimizer may mangle constant columns (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d770d8d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d770d8d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d770d8d
Branch: refs/heads/master
Commit: 2d770d8dea9670c51b9cb2a8eca5cdc3ae55e6c8
Parents: 3660ac2
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Apr 2 13:39:31 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Apr 2 13:39:45 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/optimizer/StatsOptimizer.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2d770d8d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index e490627..d26a48b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -320,7 +320,7 @@ public class StatsOptimizer extends Transform {
}
Operator<?> last = (Operator<?>) stack.get(5);
SelectOperator cselOp = null;
- Map<Integer,Object> posToConstant = new HashMap<>();
+ Map<Integer,Object> posToConstant = new LinkedHashMap<>();
if (last instanceof SelectOperator) {
cselOp = (SelectOperator) last;
if (!cselOp.isIdentitySelect()) {
@@ -338,6 +338,17 @@ public class StatsOptimizer extends Transform {
}
}
last = (Operator<?>) stack.get(6);
+ } else {
+ // Add constants if there is no SELECT on top
+ GroupByDesc gbyDesc = cgbyOp.getConf();
+ int numCols = gbyDesc.getOutputColumnNames().size();
+ int aggCols = gbyDesc.getAggregators().size();
+ List<String> dpCols = cgbyOp.getSchema().getColumnNames().subList(0, numCols - aggCols);
+ for(int i = 0; i < dpCols.size(); i++) {
+ ExprNodeDesc end = ExprNodeDescUtils.findConstantExprOrigin(dpCols.get(i), cgbyOp);
+ assert end instanceof ExprNodeConstantDesc;
+ posToConstant.put(i, ((ExprNodeConstantDesc)end).getValue());
+ }
}
FileSinkOperator fsOp = (FileSinkOperator)last;
if (fsOp.getNumChild() > 0) {
@@ -707,7 +718,10 @@ public class StatsOptimizer extends Transform {
List<String> colNames = new ArrayList<String>();
List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
if (cselOp == null) {
- allRows.add(oneRow);
+ List<Object> oneRowWithConstant = new ArrayList<>();
+ oneRowWithConstant.addAll(posToConstant.values());
+ oneRowWithConstant.addAll(oneRow);
+ allRows.add(oneRowWithConstant);
for (ColumnInfo colInfo : cgbyOp.getSchema().getSignature()) {
colNames.add(colInfo.getInternalName());
ois.add(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(colInfo.getType()));