You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/03/13 03:54:52 UTC
svn commit: r1577023 - in /pig/branches/branch-0.12: ./
src/org/apache/pig/newplan/logical/relational/
src/org/apache/pig/newplan/logical/rules/ test/org/apache/pig/test/
Author: daijy
Date: Thu Mar 13 02:54:51 2014
New Revision: 1577023
URL: http://svn.apache.org/r1577023
Log:
PIG-3782: PushDownForEachFlatten + ColumnMapKeyPrune with user defined schema failing due to incorrect UID assignment
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
pig/branches/branch-0.12/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1577023&r1=1577022&r2=1577023&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Thu Mar 13 02:54:51 2014
@@ -34,6 +34,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3782: PushDownForEachFlatten + ColumnMapKeyPrune with user defined schema failing due to incorrect UID assignment (knoguchi via daijy)
+
PIG-3779: Assert constructs ConstantExpression with null when no comment is given (thedatachef via cheolsoo)
PIG-3777: Pig 12.0 Documentation (karinahauser via daijy)
Modified: pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1577023&r1=1577022&r2=1577023&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/relational/LOGenerate.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Thu Mar 13 02:54:51 2014
@@ -33,6 +33,8 @@ import org.apache.pig.newplan.logical.re
public class LOGenerate extends LogicalRelationalOperator {
private List<LogicalExpressionPlan> outputPlans;
private boolean[] flattenFlags;
+ // mUserDefinedSchema is the original input from the user, we don't suppose
+ // to store uid in mUserDefinedSchema
private List<LogicalSchema> mUserDefinedSchema = null;
private List<LogicalSchema> outputPlanSchemas = null;
// If LOGenerate generate new uid, cache it here.
Modified: pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java?rev=1577023&r1=1577022&r2=1577023&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java Thu Mar 13 02:54:51 2014
@@ -260,6 +260,7 @@ public class PushDownForEachFlatten exte
fieldsToBeFlattaned.add(fieldCount);
if (gen.getUserDefinedSchema()!=null && gen.getUserDefinedSchema().get(i)!=null) {
cachedUserDefinedSchema.put(fieldCount, gen.getUserDefinedSchema().get(i));
+ cachedUserDefinedSchema.get(fieldCount).mergeUid(gen.getOutputPlanSchemas().get(i));
gen.getUserDefinedSchema().set(i, null);
}
fieldCount++;
Modified: pig/branches/branch-0.12/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1577023&r1=1577022&r2=1577023&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Thu Mar 13 02:54:51 2014
@@ -1111,6 +1111,25 @@ public class TestNewPlanPushDownForeachF
((LOLoad)load).getSchema().getField("a1") != null );
}
+ // See PIG-3782
+ @Test
+ public void testForeachJoinWithUserDefinedSchemaAndPruning() throws Exception {
+ String query =
+ "a = load '1.txt' as (a0:int, a1, a2:bag{});" +
+ "b = load '2.txt' as (b0:int, b1);" +
+ "c = foreach a generate a0, flatten(a2) as (q1, q2);" +
+ "d = join c by a0, b by b0;" +
+ "e = foreach d generate a0, q1, q2;" +
+ "f = foreach e generate a0, (int)q1, (int)q2;" +
+ "store f into 'output';" ;
+
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlanWithPruning( query );
+ //In the original issue, Exception is thrown during the Pruning due to
+ //incorrect UID assignment by the PushDownForEachFlatten failing this
+ //test
+
+ }
+
public class MyPlanOptimizerWithPruning extends LogicalPlanOptimizer {
protected MyPlanOptimizerWithPruning (OperatorPlan p, int iterations) {
super(p, iterations, new HashSet<String>());