You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/11/01 17:43:42 UTC
svn commit: r1029739 - in /pig/branches/branch-0.8: CHANGES.txt
src/org/apache/pig/newplan/logical/rules/MergeForEach.java
test/org/apache/pig/test/TestMergeForEachOptimization.java
Author: daijy
Date: Mon Nov 1 16:43:41 2010
New Revision: 1029739
URL: http://svn.apache.org/viewvc?rev=1029739&view=rev
Log:
PIG-1695: MergeForEach does not carry user defined schema if any one of the merged ForEach has user defined schema
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/MergeForEach.java
pig/branches/branch-0.8/test/org/apache/pig/test/TestMergeForEachOptimization.java
Modified: pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1029739&r1=1029738&r2=1029739&view=diff
==============================================================================
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Mon Nov 1 16:43:41 2010
@@ -204,6 +204,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1695: MergeForEach does not carry user defined schema if any one of the merged ForEach has user defined schema (daijy)
+
PIG-1684: Inconsistent usage of store func. (thejas)
PIG-1694: union-onschema projects null schema at parsing stage for some queries (thejas)
Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/MergeForEach.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/MergeForEach.java?rev=1029739&r1=1029738&r2=1029739&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/MergeForEach.java (original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/MergeForEach.java Mon Nov 1 16:43:41 2010
@@ -36,6 +36,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.optimizer.Rule;
import org.apache.pig.newplan.optimizer.Transformer;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -88,6 +89,13 @@ public class MergeForEach extends Rule {
if( flatten )
return false;
}
+ if (gen1.getUserDefinedSchema()!=null) {
+ for (LogicalSchema s : gen1.getUserDefinedSchema()) {
+ if (s!=null) {
+ return false;
+ }
+ }
+ }
// Check if non of the 1st foreach output is referred more than once in second foreach.
// Otherwise, we may do expression calculation more than once, defeat the benefit of this
@@ -194,6 +202,7 @@ public class MergeForEach extends Rule {
newForEach.setRequestedParallelism(foreach1.getRequestedParallelisam());
List<LogicalExpressionPlan> newExpList = new ArrayList<LogicalExpressionPlan>();
LOGenerate newGen = new LOGenerate(newForEachInnerPlan, newExpList, gen2.getFlattenFlags());
+ newGen.setUserDefinedSchema(gen2.getUserDefinedSchema());
newForEachInnerPlan.add(newGen);
for (LogicalExpressionPlan exp2 : gen2.getOutputPlans()) {
Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestMergeForEachOptimization.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestMergeForEachOptimization.java?rev=1029739&r1=1029738&r2=1029739&view=diff
==============================================================================
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestMergeForEachOptimization.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestMergeForEachOptimization.java Mon Nov 1 16:43:41 2010
@@ -32,6 +32,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.rules.AddForEach;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
import org.apache.pig.newplan.logical.rules.MergeForEach;
@@ -79,8 +80,8 @@ public class TestMergeForEachOptimizatio
public void testSimple() throws IOException {
LogicalPlanTester lpt = new LogicalPlanTester( pc );
lpt.buildPlan( "A = load 'file.txt' as (a, b, c);" );
- lpt.buildPlan( "B = foreach A generate a+b as u, c-b as v;" );
- lpt.buildPlan( "C = foreach B generate $0+5, v;" );
+ lpt.buildPlan( "B = foreach A generate a+b, c-b;" );
+ lpt.buildPlan( "C = foreach B generate $0+5, $1;" );
org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan( "store C into 'empty';" );
LogicalPlan newLogicalPlan = migratePlan( plan );
@@ -109,8 +110,8 @@ public class TestMergeForEachOptimizatio
public void testComplex() throws IOException {
LogicalPlanTester lpt = new LogicalPlanTester( pc );
lpt.buildPlan( "A = load 'file.txt' as (a:int, b, c:bag{t:tuple(c0:int,c1:int)});" );
- lpt.buildPlan( "B = foreach A { S = ORDER c BY $0; generate $0 as u, COUNT(S) as v, SUM(S) as w; };" );
- lpt.buildPlan( "C = foreach B generate w+5 as x, u-v/2;" );
+ lpt.buildPlan( "B = foreach A { S = ORDER c BY $0; generate $0, COUNT(S), SUM(S); };" );
+ lpt.buildPlan( "C = foreach B generate $2+5 as x, $0-$1/2 as y;" );
org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan( "store C into 'empty';" );
LogicalPlan newLogicalPlan = migratePlan( plan );
@@ -130,6 +131,9 @@ public class TestMergeForEachOptimizatio
Assert.assertTrue( outputExprCount1 == outputExprCount2 );
LOForEach foreach2 = getForEachOperator( newLogicalPlan );
Assert.assertTrue( foreach2.getAlias().equals( "C" ) );
+ LogicalSchema newSchema = foreach2.getSchema();
+ Assert.assertTrue(newSchema.getField(0).alias.equals("x"));
+ Assert.assertTrue(newSchema.getField(1).alias.equals("y"));
}
/**
@@ -227,9 +231,9 @@ public class TestMergeForEachOptimizatio
LogicalPlanTester lpt = new LogicalPlanTester( pc );
lpt.buildPlan( "l1 = load 'y' as (a);" );
lpt.buildPlan( "l2 = load 'z' as (a1,b1,c1,d1);" );
- lpt.buildPlan( "f1 = foreach l2 generate a1 as a, b1 as b, c1 as c, d1 as d;" );
- lpt.buildPlan( "f2 = foreach f1 generate a,b,c;" );
- lpt.buildPlan( "j1 = join f2 by a, l1 by a using 'replicated';" );
+ lpt.buildPlan( "f1 = foreach l2 generate a1, b1, c1, d1;" );
+ lpt.buildPlan( "f2 = foreach f1 generate a1, b1, c1;" );
+ lpt.buildPlan( "j1 = join f2 by a1, l1 by a using 'replicated';" );
org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan( "store j1 into 'empty';" );
LogicalPlan newLogicalPlan = migratePlan( plan );