You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/04/14 22:56:47 UTC
svn commit: r1587316 - in
/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez:
POLocalRearrangeTez.java TezCompiler.java TezDagBuilder.java
Author: cheolsoo
Date: Mon Apr 14 20:56:47 2014
New Revision: 1587316
URL: http://svn.apache.org/r1587316
Log:
PIG-3893: Hash join followed by replicated join fails in Tez mode (cheolsoo)
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java?rev=1587316&r1=1587315&r2=1587316&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/POLocalRearrangeTez.java Mon Apr 14 20:56:47 2014
@@ -48,6 +48,7 @@ public class POLocalRearrangeTez extends
protected String outputKey;
protected transient KeyValueWriter writer;
+ protected boolean isFRJoin = false;
protected boolean isSkewedJoin = false;
public POLocalRearrangeTez(OperatorKey k) {
@@ -75,6 +76,14 @@ public class POLocalRearrangeTez extends
this.outputKey = outputKey;
}
+ public boolean isFRJoin() {
+ return isFRJoin;
+ }
+
+ public void setFRJoin(boolean isFRJoin) {
+ this.isFRJoin = isFRJoin;
+ }
+
public boolean isSkewedJoin() {
return isSkewedJoin;
}
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java?rev=1587316&r1=1587315&r2=1587316&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezCompiler.java Mon Apr 14 20:56:47 2014
@@ -668,6 +668,7 @@ public class TezCompiler extends PhyPlan
if (!tezOp.isClosed()) {
POLocalRearrangeTez lr = new POLocalRearrangeTez(op.getLRs()[i]);
lr.setOutputKey(curTezOp.getOperatorKey().toString());
+ lr.setFRJoin(true);
tezOp.plan.addAsLeaf(lr);
TezEdgeDescriptor edge = TezCompilerUtil.connect(tezPlan, tezOp, curTezOp);
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1587316&r1=1587315&r2=1587316&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java Mon Apr 14 20:56:47 2014
@@ -457,7 +457,9 @@ public class TezDagBuilder extends TezOp
LinkedList<POLocalRearrangeTez> lrs =
PlanHelper.getPhysicalOperators(pred.plan, POLocalRearrangeTez.class);
for (POLocalRearrangeTez lr : lrs) {
- if (lr.getOutputKey().equals(tezOp.getOperatorKey().toString())) {
+ if (lr.isFRJoin()) {
+ // skip FR join input
+ } else if (lr.getOutputKey().equals(tezOp.getOperatorKey().toString())) {
localRearrangeMap.put((int)lr.getIndex(), inputKey);
}
}