You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/02/12 16:38:36 UTC
svn commit: r1730045 - in /pig/trunk:
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/
src/org/apache/pig/tools/pigstats/tez/
test/org/apache/pig/test/data/GoldenFiles/tez/ test/org/apache/pig/tez/
Author: rohini
Date: Fri Feb 12 15:38:35 2016
New Revision: 1730045
URL: http://svn.apache.org/viewvc?rev=1730045&view=rev
Log:
Fix TestMultiQueryLocal.testStoreOrder test failure for PIG-4759
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java
pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java Fri Feb 12 15:38:35 2016
@@ -31,6 +31,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.OperatorPlan;
import org.apache.pig.impl.plan.PlanException;
@@ -152,38 +153,109 @@ public class TezPlanContainer extends Op
}
public void split(TezPlanContainerNode planNode) throws PlanException {
+
TezOperPlan tezOperPlan = planNode.getTezOperPlan();
+ if (tezOperPlan.size() == 1) {
+ // Nothing to split. Only one operator in the plan
+ return;
+ }
+
TezOperator operToSegment = null;
List<TezOperator> succs = new ArrayList<TezOperator>();
- for (TezOperator tezOper : tezOperPlan) {
- if (tezOper.needSegmentBelow() && tezOperPlan.getSuccessors(tezOper)!=null) {
- operToSegment = tezOper;
- succs.addAll(tezOperPlan.getSuccessors(tezOper));
- break;
- }
+ try {
+ // Split top down from root to leaves
+ SegmentOperatorFinder finder = new SegmentOperatorFinder(tezOperPlan);
+ finder.visit();
+ operToSegment = finder.getOperatorToSegment();
+ } catch (VisitorException e) {
+ throw new PlanException(e);
}
- if (operToSegment != null) {
+
+ if (operToSegment != null && tezOperPlan.getSuccessors(operToSegment) != null) {
+ succs.addAll(tezOperPlan.getSuccessors(operToSegment));
for (TezOperator succ : succs) {
tezOperPlan.disconnect(operToSegment, succ);
- TezOperPlan newOperPlan = new TezOperPlan();
- List<TezPlanContainerNode> containerSuccs = new ArrayList<TezPlanContainerNode>();
- if (getSuccessors(planNode)!=null) {
- containerSuccs.addAll(getSuccessors(planNode));
- }
- tezOperPlan.moveTree(succ, newOperPlan);
- TezPlanContainerNode newPlanNode = new TezPlanContainerNode(generateNodeOperatorKey(), newOperPlan);
- add(newPlanNode);
- for (TezPlanContainerNode containerNodeSucc : containerSuccs) {
- disconnect(planNode, containerNodeSucc);
- connect(newPlanNode, containerNodeSucc);
+ }
+ for (TezOperator succ : succs) {
+ try {
+ if (tezOperPlan.getOperator(succ.getOperatorKey()) == null) {
+ // Has already been moved to a new plan by previous successor
+ // as part of dependency. It could have been further split.
+ // So walk the full plan to find the new plan and connect
+ TezOperatorFinder finder = new TezOperatorFinder(this, succ);
+ finder.visit();
+ connect(planNode, finder.getPlanContainerNode());
+ continue;
+ }
+ TezOperPlan newOperPlan = new TezOperPlan();
+ tezOperPlan.moveTree(succ, newOperPlan);
+ TezPlanContainerNode newPlanNode = new TezPlanContainerNode(
+ generateNodeOperatorKey(), newOperPlan);
+ add(newPlanNode);
+ connect(planNode, newPlanNode);
+ split(newPlanNode);
+ if (newPlanNode.getTezOperPlan().getOperator(succ.getOperatorKey()) == null) {
+ // On further split, the successor moved to a new plan container.
+ // Connect to that
+ TezOperatorFinder finder = new TezOperatorFinder(this, succ);
+ finder.visit();
+ disconnect(planNode, newPlanNode);
+ connect(planNode, finder.getPlanContainerNode());
+ }
+ } catch (VisitorException e) {
+ throw new PlanException(e);
}
- connect(planNode, newPlanNode);
- split(newPlanNode);
}
split(planNode);
}
}
+ private static class SegmentOperatorFinder extends TezOpPlanVisitor {
+
+ private TezOperator operToSegment;
+
+ public SegmentOperatorFinder(TezOperPlan plan) {
+ super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan));
+ }
+
+ public TezOperator getOperatorToSegment() {
+ return operToSegment;
+ }
+
+ @Override
+ public void visitTezOp(TezOperator tezOperator) throws VisitorException {
+ if (tezOperator.needSegmentBelow() && operToSegment == null) {
+ operToSegment = tezOperator;
+ }
+ }
+
+ }
+
+ private static class TezOperatorFinder extends TezPlanContainerVisitor {
+
+ private TezPlanContainerNode planContainerNode;
+ private TezOperator operatorToFind;
+
+ public TezOperatorFinder(TezPlanContainer plan, TezOperator operatorToFind) {
+ super(plan, new DependencyOrderWalker<TezPlanContainerNode, TezPlanContainer>(plan));
+ this.operatorToFind = operatorToFind;
+ }
+
+ public TezPlanContainerNode getPlanContainerNode() {
+ return planContainerNode;
+ }
+
+ @Override
+ public void visitTezPlanContainerNode(
+ TezPlanContainerNode tezPlanContainerNode)
+ throws VisitorException {
+ if (tezPlanContainerNode.getTezOperPlan().getOperatorKey(operatorToFind) != null) {
+ planContainerNode = tezPlanContainerNode;
+ }
+ }
+
+ }
+
private synchronized OperatorKey generateNodeOperatorKey() {
OperatorKey opKey = new OperatorKey(jobName + "-" + dagId + "_scope", scopeId);
scopeId++;
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainerPrinter.java Fri Feb 12 15:38:35 2016
@@ -19,7 +19,8 @@ package org.apache.pig.backend.hadoop.ex
import java.io.PrintStream;
-import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezGraphPrinter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezDAGGraphPrinter;
+import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezPrinter.TezVertexGraphPrinter;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.VisitorException;
@@ -37,6 +38,19 @@ public class TezPlanContainerPrinter ext
mStream.println("#--------------------------------------------------");
mStream.println("# There are " + planContainer.size() + " DAGs in the session");
mStream.println("#--------------------------------------------------");
+ printContainerPlan(planContainer);
+ }
+
+ private void printContainerPlan(TezPlanContainer planContainer) {
+ try {
+ if (planContainer.size() > 1) {
+ TezDAGGraphPrinter graphPrinter = new TezDAGGraphPrinter(planContainer);
+ graphPrinter.visit();
+ mStream.print(graphPrinter.toString());
+ }
+ } catch (VisitorException e) {
+ throw new RuntimeException(e);
+ }
}
public void setVerbose(boolean verbose) {
@@ -48,7 +62,7 @@ public class TezPlanContainerPrinter ext
mStream.println("#--------------------------------------------------");
mStream.println("# TEZ DAG plan: " + tezPlanContainerNode.getOperatorKey());
mStream.println("#--------------------------------------------------");
- TezGraphPrinter graphPrinter = new TezGraphPrinter(tezPlanContainerNode.getTezOperPlan());
+ TezVertexGraphPrinter graphPrinter = new TezVertexGraphPrinter(tezPlanContainerNode.getTezOperPlan());
graphPrinter.visit();
mStream.print(graphPrinter.toString());
TezPrinter printer = new TezPrinter(mStream, tezPlanContainerNode.getTezOperPlan());
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPrinter.java Fri Feb 12 15:38:35 2016
@@ -99,11 +99,11 @@ public class TezPrinter extends TezOpPla
/**
* This class prints the Tez Vertex Graph
*/
- public static class TezGraphPrinter extends TezOpPlanVisitor {
+ public static class TezVertexGraphPrinter extends TezOpPlanVisitor {
StringBuilder buf;
- public TezGraphPrinter(TezOperPlan plan) {
+ public TezVertexGraphPrinter(TezOperPlan plan) {
super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan, true));
buf = new StringBuilder();
}
@@ -132,6 +132,43 @@ public class TezPrinter extends TezOpPla
}
}
buf.append("\n");
+ }
+
+ @Override
+ public String toString() {
+ buf.append("\n");
+ return buf.toString();
+ }
+ }
+
+ /**
+ * This class prints the Tez DAG Graph
+ */
+ public static class TezDAGGraphPrinter extends TezPlanContainerVisitor {
+
+ StringBuilder buf;
+
+ public TezDAGGraphPrinter(TezPlanContainer plan) {
+ super(plan, new DependencyOrderWalker<TezPlanContainerNode, TezPlanContainer>(plan, true));
+ buf = new StringBuilder();
+ }
+
+ @Override
+ public void visitTezPlanContainerNode(TezPlanContainerNode tezPlanContainerNode) throws VisitorException {
+ writePlan(mPlan, tezPlanContainerNode, buf);
+ }
+
+ public static void writePlan(TezPlanContainer mPlan, TezPlanContainerNode tezPlanContainerNode, StringBuilder buf) {
+ buf.append("Tez DAG " + tezPlanContainerNode.getOperatorKey().toString());
+ List<TezPlanContainerNode> succs = mPlan.getSuccessors(tezPlanContainerNode);
+ if (succs != null) {
+ Collections.sort(succs);
+ buf.append("\t->\t");
+ for (TezPlanContainerNode op : succs) {
+ buf.append("Tez DAG " + op.getOperatorKey().toString()).append(",");
+ }
+ }
+ buf.append("\n");
}
@Override
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/tez/TezDAGStats.java Fri Feb 12 15:38:35 2016
@@ -139,7 +139,7 @@ public class TezDAGStats extends JobStat
@Override
public void visitTezOp(TezOperator tezOp) throws VisitorException {
- TezPrinter.TezGraphPrinter.writePlan(mPlan, tezOp, tezDAGPlan);
+ TezPrinter.TezVertexGraphPrinter.writePlan(mPlan, tezOp, tezDAGPlan);
TezVertexStats currStats =
new TezVertexStats(tezOp.getOperatorKey().toString(), jobPlan, tezOp.isUseMRMapSettings());
jobPlan.add(currStats);
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-1.gld Fri Feb 12 15:38:35 2016
@@ -1,6 +1,9 @@
#--------------------------------------------------
# There are 2 DAGs in the session
#--------------------------------------------------
+Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1,
+Tez DAG pig-1_scope-1
+
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld?rev=1730045&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld Fri Feb 12 15:38:35 2016
@@ -0,0 +1,83 @@
+#--------------------------------------------------
+# There are 5 DAGs in the session
+#--------------------------------------------------
+Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1,Tez DAG pig-2_scope-2,Tez DAG pig-3_scope-3,
+Tez DAG pig-2_scope-2 -> Tez DAG pig-3_scope-3,Tez DAG pig-4_scope-4,
+Tez DAG pig-1_scope-1 -> Tez DAG pig-3_scope-3,
+Tez DAG pig-3_scope-3
+Tez DAG pig-4_scope-4
+
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-20
+
+Tez vertex scope-20
+# Plan on vertex
+a: Store(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-1
+|
+|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+#--------------------------------------------------
+# TEZ DAG plan: pig-2_scope-2
+#--------------------------------------------------
+Tez vertex scope-21
+
+Tez vertex scope-21
+# Plan on vertex
+a: Store(file:///tmp/output/Dir2:BinStorage) - scope-3
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-2
+#--------------------------------------------------
+# TEZ DAG plan: pig-1_scope-1
+#--------------------------------------------------
+Tez vertex scope-23
+
+Tez vertex scope-23
+# Plan on vertex
+a: Store(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-7
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-6
+#--------------------------------------------------
+# TEZ DAG plan: pig-3_scope-3
+#--------------------------------------------------
+Tez vertex scope-24 -> Tez vertex scope-27,
+Tez vertex scope-26 -> Tez vertex scope-27,
+Tez vertex scope-25 -> Tez vertex scope-27,
+Tez vertex scope-27
+
+Tez vertex scope-24
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-13 -> scope-27
+| |
+| Project[bytearray][0] - scope-14
+|
+|---a: Load(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-26
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-17 -> scope-27
+| |
+| Project[bytearray][0] - scope-18
+|
+|---c: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-10
+Tez vertex scope-25
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-15 -> scope-27
+| |
+| Project[bytearray][0] - scope-16
+|
+|---b: Load(file:///tmp/output/Dir2:BinStorage) - scope-9
+Tez vertex scope-27
+# Plan on vertex
+d: Store(file:///tmp/output/Dir5:org.apache.pig.builtin.PigStorage) - scope-19
+|
+|---d: Package(Packager)[tuple]{bytearray} - scope-12
+#--------------------------------------------------
+# TEZ DAG plan: pig-4_scope-4
+#--------------------------------------------------
+Tez vertex scope-22
+
+Tez vertex scope-22
+# Plan on vertex
+a: Store(file:///tmp/output/Dir4:org.apache.pig.builtin.PigStorage) - scope-5
+|
+|---a: Load(file:///tmp/output/Dir2:BinStorage) - scope-4
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld?rev=1730045&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld Fri Feb 12 15:38:35 2016
@@ -0,0 +1,83 @@
+#--------------------------------------------------
+# There are 5 DAGs in the session
+#--------------------------------------------------
+Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1,Tez DAG pig-3_scope-3,Tez DAG pig-4_scope-4,
+Tez DAG pig-1_scope-1 -> Tez DAG pig-2_scope-2,Tez DAG pig-4_scope-4,
+Tez DAG pig-2_scope-2
+Tez DAG pig-3_scope-3 -> Tez DAG pig-4_scope-4,
+Tez DAG pig-4_scope-4
+
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-20
+
+Tez vertex scope-20
+# Plan on vertex
+a: Store(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-1
+|
+|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+#--------------------------------------------------
+# TEZ DAG plan: pig-1_scope-1
+#--------------------------------------------------
+Tez vertex scope-21
+
+Tez vertex scope-21
+# Plan on vertex
+a: Store(file:///tmp/output/Dir2:BinStorage) - scope-3
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-2
+#--------------------------------------------------
+# TEZ DAG plan: pig-2_scope-2
+#--------------------------------------------------
+Tez vertex scope-22
+
+Tez vertex scope-22
+# Plan on vertex
+a: Store(file:///tmp/output/Dir4:org.apache.pig.builtin.PigStorage) - scope-5
+|
+|---a: Load(file:///tmp/output/Dir2:BinStorage) - scope-4
+#--------------------------------------------------
+# TEZ DAG plan: pig-3_scope-3
+#--------------------------------------------------
+Tez vertex scope-23
+
+Tez vertex scope-23
+# Plan on vertex
+a: Store(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-7
+|
+|---a: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-6
+#--------------------------------------------------
+# TEZ DAG plan: pig-4_scope-4
+#--------------------------------------------------
+Tez vertex scope-26 -> Tez vertex scope-27,
+Tez vertex scope-25 -> Tez vertex scope-27,
+Tez vertex scope-24 -> Tez vertex scope-27,
+Tez vertex scope-27
+
+Tez vertex scope-26
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-17 -> scope-27
+| |
+| Project[bytearray][0] - scope-18
+|
+|---c: Load(file:///tmp/output/Dir1:org.apache.pig.builtin.PigStorage) - scope-10
+Tez vertex scope-25
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-15 -> scope-27
+| |
+| Project[bytearray][0] - scope-16
+|
+|---b: Load(file:///tmp/output/Dir2:BinStorage) - scope-9
+Tez vertex scope-24
+# Plan on vertex
+d: Local Rearrange[tuple]{bytearray}(false) - scope-13 -> scope-27
+| |
+| Project[bytearray][0] - scope-14
+|
+|---a: Load(file:///tmp/output/Dir3:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-27
+# Plan on vertex
+d: Store(file:///tmp/output/Dir5:org.apache.pig.builtin.PigStorage) - scope-19
+|
+|---d: Package(Packager)[tuple]{bytearray} - scope-12
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Native-1.gld Fri Feb 12 15:38:35 2016
@@ -1,6 +1,10 @@
#--------------------------------------------------
# There are 3 DAGs in the session
#--------------------------------------------------
+Tez DAG pig-0_scope-0 -> Tez DAG pig-1_scope-1,
+Tez DAG pig-1_scope-1 -> Tez DAG pig-2_scope-2,
+Tez DAG pig-2_scope-2
+
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1730045&r1=1730044&r2=1730045&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Fri Feb 12 15:38:35 2016
@@ -100,6 +100,31 @@ public class TestTezCompiler {
}
@Test
+ public void testStoreLoadMultiple() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input';" +
+ "store a into 'file:///tmp/output/Dir1';" +
+ "a = load 'file:///tmp/output/Dir1';" +
+ "store a into 'file:///tmp/output/Dir2' using BinStorage();" +
+ "a = load 'file:///tmp/output/Dir1';" +
+ "store a into 'file:///tmp/output/Dir3';" +
+ "a = load 'file:///tmp/output/Dir2' using BinStorage();" +
+ "store a into 'file:///tmp/output/Dir4';" +
+ "a = load 'file:///tmp/output/Dir3';" +
+ "b = load 'file:///tmp/output/Dir2' using BinStorage();" +
+ "c = load 'file:///tmp/output/Dir1';" +
+ "d = cogroup a by $0, b by $0, c by $0;" +
+ "store d into 'file:///tmp/output/Dir5';";
+
+ // To get around difference in ordering of operators in plan due to JDK7 and JDK8
+ if (System.getProperties().getProperty("java.version").startsWith("1.8")) {
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2.gld");
+ } else {
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-LoadStore-2-JDK7.gld");
+ }
+ }
+
+ @Test
public void testNative() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +