You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/10/24 04:21:12 UTC
svn commit: r1633987 - in
/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql:
optimizer/ConvertJoinMapJoin.java optimizer/MergeJoinProc.java
parse/GenTezWork.java
Author: gunther
Date: Fri Oct 24 02:21:12 2014
New Revision: 1633987
URL: http://svn.apache.org/r1633987
Log:
HIVE-8563: Running annotate_stats_join_pkfk.q in TestMiniTezCliDriver is causing NPE (Vikram Dixit K via Gunther Hagleitner)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1633987&r1=1633986&r2=1633987&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Fri Oct 24 02:21:12 2014
@@ -228,7 +228,7 @@ public class ConvertJoinMapJoin implemen
@SuppressWarnings("unchecked")
CommonMergeJoinOperator mergeJoinOp =
(CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets,
- isSubQuery, mapJoinConversionPos, mapJoinDesc));
+ isSubQuery, mapJoinConversionPos, mapJoinDesc), joinOp.getSchema());
OpTraits opTraits =
new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits()
.getSortCols());
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java?rev=1633987&r1=1633986&r2=1633987&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java Fri Oct 24 02:21:12 2014
@@ -22,21 +22,6 @@ import org.apache.hadoop.hive.ql.plan.Te
import org.apache.hadoop.hive.ql.plan.TezWork.VertexType;
public class MergeJoinProc implements NodeProcessor {
-
- public Operator<? extends OperatorDesc> getLeafOperator(Operator<? extends OperatorDesc> op) {
- for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) {
- // FileSink or ReduceSink operators are used to create vertices. See
- // TezCompiler.
- if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof FileSinkOperator)) {
- return childOp;
- } else {
- return getLeafOperator(childOp);
- }
- }
-
- return null;
- }
-
@Override
public Object
process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
@@ -60,13 +45,13 @@ public class MergeJoinProc implements No
// merge work already exists for this merge join operator, add the dummy store work to the
// merge work. Else create a merge work, add above work to the merge work
MergeJoinWork mergeWork = null;
- if (context.opMergeJoinWorkMap.containsKey(getLeafOperator(mergeJoinOp))) {
+ if (context.opMergeJoinWorkMap.containsKey(mergeJoinOp)) {
// we already have the merge work corresponding to this merge join operator
- mergeWork = context.opMergeJoinWorkMap.get(getLeafOperator(mergeJoinOp));
+ mergeWork = context.opMergeJoinWorkMap.get(mergeJoinOp);
} else {
mergeWork = new MergeJoinWork();
tezWork.add(mergeWork);
- context.opMergeJoinWorkMap.put(getLeafOperator(mergeJoinOp), mergeWork);
+ context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork);
}
mergeWork.setMergeJoinOperator(mergeJoinOp);
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1633987&r1=1633986&r2=1633987&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java Fri Oct 24 02:21:12 2014
@@ -137,15 +137,15 @@ public class GenTezWork implements NodeP
// we are currently walking the big table side of the merge join. we need to create or hook up
// merge join work.
MergeJoinWork mergeJoinWork = null;
- if (context.opMergeJoinWorkMap.containsKey(operator)) {
+ if (context.opMergeJoinWorkMap.containsKey(context.currentMergeJoinOperator)) {
// we have found a merge work corresponding to this closing operator. Hook up this work.
- mergeJoinWork = context.opMergeJoinWorkMap.get(operator);
+ mergeJoinWork = context.opMergeJoinWorkMap.get(context.currentMergeJoinOperator);
} else {
// we need to create the merge join work
mergeJoinWork = new MergeJoinWork();
mergeJoinWork.setMergeJoinOperator(context.currentMergeJoinOperator);
tezWork.add(mergeJoinWork);
- context.opMergeJoinWorkMap.put(operator, mergeJoinWork);
+ context.opMergeJoinWorkMap.put(context.currentMergeJoinOperator, mergeJoinWork);
}
// connect the work correctly.
mergeJoinWork.addMergedWork(work, null);
@@ -334,10 +334,15 @@ public class GenTezWork implements NodeP
UnionWork unionWork = (UnionWork) followingWork;
int index = getMergeIndex(tezWork, unionWork, rs);
// guaranteed to be instance of MergeJoinWork if index is valid
- MergeJoinWork mergeJoinWork = (MergeJoinWork) tezWork.getChildren(unionWork).get(index);
- // disconnect the connection to union work and connect to merge work
- followingWork = mergeJoinWork;
- rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
+ if (baseWork instanceof MergeJoinWork) {
+ MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+ // disconnect the connection to union work and connect to merge work
+ followingWork = mergeJoinWork;
+ rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ } else {
+ rWork = (ReduceWork) baseWork;
+ }
} else {
rWork = (ReduceWork) followingWork;
}