You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/05/08 19:06:58 UTC
svn commit: r1678396 - in /pig/branches/branch-0.15: ./
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/fetch/
test/org/apache/pig/test/
Author: daijy
Date: Fri May 8 17:06:57 2015
New Revision: 1678396
URL: http://svn.apache.org/r1678396
Log:
PIG-4538: Pig script fail with CNF in follow up MR job
Modified:
pig/branches/branch-0.15/CHANGES.txt
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java
pig/branches/branch-0.15/test/org/apache/pig/test/TestMRCompiler.java
Modified: pig/branches/branch-0.15/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1678396&r1=1678395&r2=1678396&view=diff
==============================================================================
--- pig/branches/branch-0.15/CHANGES.txt (original)
+++ pig/branches/branch-0.15/CHANGES.txt Fri May 8 17:06:57 2015
@@ -66,6 +66,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4538: Pig script fail with CNF in follow up MR job (daijy)
+
PIG-4537: Fix unit test failure introduced by TEZ-2392: TestCollectedGroup, TestLimitVariable,
TestMapSideCogroup, etc (daijy)
Modified: pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1678396&r1=1678395&r2=1678396&view=diff
==============================================================================
--- pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Fri May 8 17:06:57 2015
@@ -42,6 +42,7 @@ import org.apache.pig.backend.hadoop.dat
import org.apache.pig.backend.hadoop.executionengine.fetch.FetchLauncher;
import org.apache.pig.backend.hadoop.executionengine.fetch.FetchOptimizer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PhyPlanSetter;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
@@ -296,6 +297,7 @@ public abstract class HExecutionEngine i
//skipped; a SimpleFetchPigStats will be returned through which the result
//can be directly fetched from the underlying storage
if (FetchOptimizer.isPlanFetchable(pc, pp)) {
+ new PhyPlanSetter(pp).visit();
return new FetchLauncher(pc).launchPig(pp);
}
return launcher.launchPig(pp, grpName, pigContext);
Modified: pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java?rev=1678396&r1=1678395&r2=1678396&view=diff
==============================================================================
--- pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java (original)
+++ pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchOptimizer.java Fri May 8 17:06:57 2015
@@ -176,7 +176,6 @@ public class FetchOptimizer {
@Override
public void visit() throws VisitorException {
- new PhyPlanSetter(mPlan).visit();
super.visit();
}
Modified: pig/branches/branch-0.15/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/test/org/apache/pig/test/TestMRCompiler.java?rev=1678396&r1=1678395&r2=1678396&view=diff
==============================================================================
--- pig/branches/branch-0.15/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/branches/branch-0.15/test/org/apache/pig/test/TestMRCompiler.java Fri May 8 17:06:57 2015
@@ -39,6 +39,7 @@ import org.apache.pig.FuncSpec;
import org.apache.pig.IndexableLoadFunc;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.fetch.FetchOptimizer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompilerException;
@@ -130,7 +131,8 @@ import org.junit.runner.RunWith;
"testUDFInMergedCoGroup",
"testUDFInMergedJoin",
"testSchemaInStoreForDistinctLimit",
- "testStorerLimit"})
+ "testStorerLimit",
+ "testFetchOptimizerSideEffect"})
public class TestMRCompiler {
static MiniCluster cluster;
@@ -1280,5 +1282,27 @@ public class TestMRCompiler {
POStore store = (POStore)firstMrOper.reducePlan.getLeaves().get(0);
assertEquals(store.getStoreFunc().getClass().getName(), "org.apache.pig.impl.io.InterStorage");
}
+
+ // See PIG-4538
+ @Test
+ public void testFetchOptimizerSideEffect() throws Exception{
+ String query = "in1 = LOAD 'data.txt' AS (ident:chararray);" +
+ "in2 = LOAD 'data.txt' AS (ident:chararray);" +
+ "in3 = LOAD 'data.txt';" +
+ "joined = JOIN in1 BY ident LEFT OUTER, in2 BY ident;" +
+ "store joined into 'output';";
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ MROperPlan mp = Util.buildMRPlan(pp, pc);
+ // isPlanFetchable should not bring side effect:
+ // set parentPlan for operators
+ FetchOptimizer.isPlanFetchable(pc, pp);
+ MapReduceOper op = mp.getLeaves().get(0);
+ PhysicalOperator store = op.reducePlan.getLeaves().get(0);
+ POForEach foreach = (POForEach)op.reducePlan.getPredecessors(store).get(0);
+ PhysicalOperator project = foreach.getInputPlans().get(0).getRoots().get(0);
+ Field parentPlan = PhysicalOperator.class.getDeclaredField("parentPlan");
+ parentPlan.setAccessible(true);
+ assertTrue(parentPlan.get(project)==null);
+ }
}