You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2013/05/28 02:59:26 UTC
svn commit: r1486722 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
test/results/compiler/plan/join1.q.xml test/results/compiler/plan/join3.q.xml
Author: navis
Date: Tue May 28 00:59:26 2013
New Revision: 1486722
URL: http://svn.apache.org/r1486722
Log:
HIVE-4572 ColumnPruner cannot preserve RS key columns corresponding to un-selected join keys in columnExprMap (Yin Huai via Navis)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1486722&r1=1486721&r2=1486722&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Tue May 28 00:59:26 2013
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
@@ -695,12 +696,14 @@ public final class ColumnPrunerProcFacto
ReduceSinkOperator reduce, ColumnPrunerProcCtx cppCtx) throws SemanticException {
ReduceSinkDesc reduceConf = reduce.getConf();
Map<String, ExprNodeDesc> oldMap = reduce.getColumnExprMap();
+ LOG.info("RS " + reduce.getIdentifier() + " oldColExprMap: " + oldMap);
RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(reduce).getRowResolver();
ArrayList<ColumnInfo> signature = oldRR.getRowSchema().getSignature();
List<String> valueColNames = reduceConf.getOutputValueColumnNames();
ArrayList<String> newValueColNames = new ArrayList<String>();
+ List<ExprNodeDesc> keyExprs = reduceConf.getKeyCols();
List<ExprNodeDesc> valueExprs = reduceConf.getValueCols();
ArrayList<ExprNodeDesc> newValueExprs = new ArrayList<ExprNodeDesc>();
@@ -713,10 +716,16 @@ public final class ColumnPrunerProcFacto
outputCol = Utilities.ReduceField.VALUE.toString() + "." + outputCol;
nm = oldRR.reverseLookup(outputCol);
}
- ColumnInfo colInfo = oldRR.getFieldMap(nm[0]).remove(nm[1]);
- oldRR.getInvRslvMap().remove(colInfo.getInternalName());
- oldMap.remove(outputCol);
- signature.remove(colInfo);
+
+ // Only remove information of a column if it is not a key,
+ // i.e. this column is not appearing in keyExprs of the RS
+ if (ExprNodeDescUtils.indexOf(outputColExpr, keyExprs) == -1) {
+ ColumnInfo colInfo = oldRR.getFieldMap(nm[0]).remove(nm[1]);
+ oldRR.getInvRslvMap().remove(colInfo.getInternalName());
+ oldMap.remove(outputCol);
+ signature.remove(colInfo);
+ }
+
} else {
newValueColNames.add(outputCol);
newValueExprs.add(outputColExpr);
@@ -729,6 +738,7 @@ public final class ColumnPrunerProcFacto
.getFieldSchemasFromColumnList(reduceConf.getValueCols(),
newValueColNames, 0, ""));
reduceConf.setValueSerializeInfo(newValueTable);
+ LOG.info("RS " + reduce.getIdentifier() + " newColExprMap: " + oldMap);
}
/**
Modified: hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml?rev=1486722&r1=1486721&r2=1486722&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml Tue May 28 00:59:26 2013
@@ -486,6 +486,20 @@
</void>
</object>
</void>
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="typeInfo">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
</object>
</void>
<void property="conf">
@@ -639,6 +653,19 @@
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
<string>VALUE._col1</string>
</void>
<void property="tabAlias">
Modified: hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml?rev=1486722&r1=1486721&r2=1486722&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml (original)
+++ hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml Tue May 28 00:59:26 2013
@@ -615,7 +615,26 @@
<void method="add">
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="columnExprMap">
- <object class="java.util.HashMap"/>
+ <object class="java.util.HashMap">
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="typeInfo">
+ <object id="PrimitiveTypeInfo0" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
+ <void property="typeName">
+ <string>string</string>
+ </void>
+ </object>
+ </void>
+ </object>
+ </void>
+ </object>
</void>
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.ReduceSinkDesc">
@@ -633,11 +652,7 @@
<string>src2</string>
</void>
<void property="typeInfo">
- <object id="PrimitiveTypeInfo0" class="org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo">
- <void property="typeName">
- <string>string</string>
- </void>
- </object>
+ <object idref="PrimitiveTypeInfo0"/>
</void>
</object>
</void>
@@ -760,7 +775,21 @@
<void property="schema">
<object class="org.apache.hadoop.hive.ql.exec.RowSchema">
<void property="signature">
- <object class="java.util.ArrayList"/>
+ <object class="java.util.ArrayList">
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src2</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ </object>
</void>
</object>
</void>
@@ -899,6 +928,20 @@
</void>
</object>
</void>
+ <void method="put">
+ <string>VALUE._col0</string>
+ <object class="org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc">
+ <void property="column">
+ <string>key</string>
+ </void>
+ <void property="tabAlias">
+ <string>src3</string>
+ </void>
+ <void property="typeInfo">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
</object>
</void>
<void property="conf">
@@ -1052,6 +1095,19 @@
<void method="add">
<object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
<void property="internalName">
+ <string>VALUE._col0</string>
+ </void>
+ <void property="tabAlias">
+ <string>src3</string>
+ </void>
+ <void property="type">
+ <object idref="PrimitiveTypeInfo0"/>
+ </void>
+ </object>
+ </void>
+ <void method="add">
+ <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo">
+ <void property="internalName">
<string>VALUE._col1</string>
</void>
<void property="tabAlias">