You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/01/12 03:30:37 UTC

svn commit: r1057974 - in /pig/trunk: ./ src/org/apache/pig/newplan/logical/rules/ src/org/apache/pig/newplan/optimizer/ test/org/apache/pig/test/

Author: daijy
Date: Wed Jan 12 02:30:37 2011
New Revision: 1057974

URL: http://svn.apache.org/viewvc?rev=1057974&view=rev
Log:
PIG-1785: New logical plan: uid conflict in flattened fields

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java
    pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java
    pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java
    pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java
    pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jan 12 02:30:37 2011
@@ -257,6 +257,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1785: New logical plan: uid conflict in flattened fields (daijy)
+
 PIG-1787: Error in logical plan generated (daijy)
 
 PIG-1791: System property mapred.output.compress, but pig-cluster-hadoop-site.xml doesn't (daijy)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java Wed Jan 12 02:30:37 2011
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.pig.FuncSpec;
+import org.apache.pig.data.DataType;
 import org.apache.pig.impl.builtin.IdentityColumn;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.Pair;
@@ -35,6 +36,8 @@ import org.apache.pig.newplan.logical.ex
 import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
 import org.apache.pig.newplan.logical.expression.ProjectExpression;
 import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
+import org.apache.pig.newplan.logical.optimizer.UidResetter;
 import org.apache.pig.newplan.logical.relational.LOCross;
 import org.apache.pig.newplan.logical.relational.LOForEach;
 import org.apache.pig.newplan.logical.relational.LOGenerate;
@@ -44,6 +47,7 @@ import org.apache.pig.newplan.logical.re
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
 import org.apache.pig.newplan.logical.rules.PushDownForEachFlatten.PushDownForEachFlattenTransformer;
 import org.apache.pig.newplan.optimizer.Rule;
 import org.apache.pig.newplan.optimizer.Transformer;
@@ -56,6 +60,8 @@ public class DuplicateForEachColumnRewri
 
     public DuplicateForEachColumnRewrite(String n) {
         super(n, true);
+        // See comments in ImplicitSplitInserter for the reason to skip listener 
+        setSkipListener(true);
     }
 
     @Override
@@ -81,18 +87,48 @@ public class DuplicateForEachColumnRewri
             LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);
             
             List<LogicalExpressionPlan> expPlans = gen.getOutputPlans();
+            boolean[] flattens = gen.getFlattenFlags();
             
             List<Long> uidSeen = new ArrayList<Long>();
             
-            for (LogicalExpressionPlan expPlan : expPlans) {
+            for (int i=0;i<expPlans.size();i++) {
+                LogicalExpressionPlan expPlan = expPlans.get(i);
+                boolean flatten = flattens[i];
                 LogicalExpression exp = (LogicalExpression)expPlan.getSources().get(0);
                 if (exp.getFieldSchema()!=null) {
-                    long uid = exp.getFieldSchema().uid;
-                    if (uidSeen.contains(uid)) {
-                        expPlansToInsertIdentity.add(expPlan);
+                    if (flatten && (exp.getFieldSchema().type == DataType.BAG || exp.getFieldSchema().type == DataType.TUPLE)) {
+                        List<LogicalFieldSchema> innerFieldSchemas = null;
+                        if (exp.getFieldSchema().type == DataType.BAG) {
+                            if (exp.getFieldSchema().schema!=null) {
+                                if (exp.getFieldSchema().schema.isTwoLevelAccessRequired()) {
+                                    //  assert(fieldSchema.schema.size() == 1 && fieldSchema.schema.getField(0).type == DataType.TUPLE)
+                                    innerFieldSchemas = exp.getFieldSchema().schema.getField(0).schema.getFields();
+                                } else {
+                                    innerFieldSchemas = exp.getFieldSchema().schema.getFields();
+                                }
+                            }
+                        }
+                        else { // DataType.TUPLE
+                            if (exp.getFieldSchema().schema!=null)
+                                innerFieldSchemas = exp.getFieldSchema().schema.getFields();
+                        }
+                        if (innerFieldSchemas != null) {
+                            for (LogicalFieldSchema innerFieldSchema : innerFieldSchemas) {
+                                long uid = innerFieldSchema.uid;
+                                if (checkAndAdd(uid, uidSeen)) {
+                                    // Seen before
+                                    expPlansToInsertIdentity.add(expPlan);
+                                    break;
+                                }
+                            }
+                        }
                     }
                     else {
-                        uidSeen.add(uid);
+                        long uid = exp.getFieldSchema().uid;
+                        if (checkAndAdd(uid, uidSeen)) {
+                            // Seen before
+                            expPlansToInsertIdentity.add(expPlan);
+                        }
                     }
                 }
             }
@@ -101,7 +137,14 @@ public class DuplicateForEachColumnRewri
                 return false;
             
             return true;
-        } 
+        }
+        
+        private boolean checkAndAdd(long uid, List<Long> uidSeen) {
+            if (uidSeen.contains(uid))
+                return true;
+            uidSeen.add(uid);
+            return false;
+        }
         
         @Override
         public OperatorPlan reportChanges() {
@@ -116,6 +159,14 @@ public class DuplicateForEachColumnRewri
                 expPlan.connect(userFuncExpression, oldRoot);
             }
             expPlansToInsertIdentity.clear();
+
+            // Since we adjust the uid layout, clear all cached uids
+            UidResetter uidResetter = new UidResetter(currentPlan);
+            uidResetter.visit();
+            
+            // Manually regenerate schema since we skip listener
+            SchemaResetter schemaResetter = new SchemaResetter(currentPlan);
+            schemaResetter.visit();
         }
     }
 }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java Wed Jan 12 02:30:37 2011
@@ -26,6 +26,7 @@ import org.apache.pig.data.DataType;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.Pair;
 import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
 import org.apache.pig.newplan.logical.optimizer.UidResetter;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.OperatorPlan;
@@ -49,6 +50,11 @@ public class ImplicitSplitInserter exten
 
     public ImplicitSplitInserter(String n) {
         super(n, true);
+        // Skip listener, especially, skip ProjectionPatcher so that we can keep column reference for 
+        // ProjectExpression (Once ProjectionPatcher is invoked, column reference will be gone in favor of uid reference).
+        // There is no need for ProjectionPatcher in this rule since we don't swap columns; however, uid conflict is not solved
+        // at this moment (until after DuplicateForEachColumnRewrite), so keep column reference for now
+        setSkipListener(true);
     }
 
     @Override
@@ -128,8 +134,14 @@ public class ImplicitSplitInserter exten
           currentPlan.connect(splitOp, splitOutput);
           currentPlan.connect(splitOutput, pos.first, suc, pos.second);
         }
+        
+        // Since we adjust the uid layout, clear all cached uids
         UidResetter uidResetter = new UidResetter(currentPlan);
         uidResetter.visit();
+
+        // Manually regenerate schema since we skip listener
+        SchemaResetter schemaResetter = new SchemaResetter(currentPlan);
+        schemaResetter.visit();
       }
       
       @Override

Modified: pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java Wed Jan 12 02:30:37 2011
@@ -108,8 +108,10 @@ public abstract class PlanOptimizer {
                                 if (transformer.check(m)) {
                                     sawMatch = true;
                                     transformer.transform(m);
-                                    for(PlanTransformListener l: listeners) {
-                                        l.transformed(plan, transformer.reportChanges());
+                                    if (!rule.isSkipListener()) {
+                                        for(PlanTransformListener l: listeners) {
+                                            l.transformed(plan, transformer.reportChanges());
+                                        }
                                     }
                                 }
                             } catch (Exception e) {

Modified: pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java Wed Jan 12 02:30:37 2011
@@ -51,6 +51,7 @@ public abstract class Rule {
     protected static final Log log = LogFactory.getLog(Rule.class);
     private transient Set<Operator> matchedNodes = new HashSet<Operator>();
     private boolean mandatory;
+    private boolean skipListener = false;
     
     /**
      * Create this rule by using the default pattern that this rule provided
@@ -95,6 +96,14 @@ public abstract class Rule {
         return pattern;
     }
     
+    protected boolean isSkipListener() {
+        return this.skipListener;
+    }
+    
+    protected void setSkipListener(boolean skip) {
+        this.skipListener = skip;
+    }
+    
     /**
      * Search for all the sub-plans that matches the pattern
      * defined by this rule. 

Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Wed Jan 12 02:30:37 2011
@@ -989,7 +989,7 @@ public class TestEvalPipeline2 extends T
     
     // See PIG-1766
     @Test
-    public void testForEachSameOriginColumn() throws Exception{
+    public void testForEachSameOriginColumn1() throws Exception{
         String[] input1 = {
                 "1\t2",
                 "1\t3",
@@ -1002,10 +1002,10 @@ public class TestEvalPipeline2 extends T
                 "2\ttwo",
         };
         
-        Util.createInputFile(cluster, "table_testForEachSameOriginColumn1", input1);
-        Util.createInputFile(cluster, "table_testForEachSameOriginColumn2", input2);
-        pigServer.registerQuery("A = load 'table_testForEachSameOriginColumn1' AS (a0:int, a1:int);");
-        pigServer.registerQuery("B = load 'table_testForEachSameOriginColumn2' AS (b0:int, b1:chararray);");
+        Util.createInputFile(cluster, "table_testForEachSameOriginColumn1_1", input1);
+        Util.createInputFile(cluster, "table_testForEachSameOriginColumn1_2", input2);
+        pigServer.registerQuery("A = load 'table_testForEachSameOriginColumn1_1' AS (a0:int, a1:int);");
+        pigServer.registerQuery("B = load 'table_testForEachSameOriginColumn1_2' AS (b0:int, b1:chararray);");
         pigServer.registerQuery("C = join A by a0, B by b0;");
         pigServer.registerQuery("D = foreach B generate b0 as d0, b1 as d1;");
         pigServer.registerQuery("E = join C by a1, D by d0;");
@@ -1156,4 +1156,46 @@ public class TestEvalPipeline2 extends T
         assertTrue(t.toString().equals("(1,2,1,2)"));
         assertFalse(iter.hasNext());
     }
+    
+    // See PIG-1785
+    @Test
+    public void testForEachSameOriginColumn2() throws Exception{
+        String[] input1 = {
+                "{(1,2),(2,3)}",
+        };
+        
+        Util.createInputFile(cluster, "table_testForEachSameOriginColumn2", input1);
+
+        pigServer.registerQuery("a = load 'table_testForEachSameOriginColumn2' as (a0:bag{t:tuple(i0:int, i1:int)});");
+        pigServer.registerQuery("b = foreach a generate flatten(a0) as (b0, b1), flatten(a0) as (b2, b3);");
+        pigServer.registerQuery("c = filter b by b0>b2;");
+        
+        Iterator<Tuple> iter = pigServer.openIterator("c");
+        
+        Tuple t = iter.next();
+        assertTrue(t.toString().contains("(2,3,1,2)"));
+        assertFalse(iter.hasNext());
+    }
+    
+    // See PIG-1785
+    @Test
+    public void testForEachSameOriginColumn3() throws Exception{
+        String[] input1 = {
+                "1\t1\t2",
+                "1\t2\t3",
+        };
+        
+        Util.createInputFile(cluster, "table_testForEachSameOriginColumn3", input1);
+
+        pigServer.registerQuery("a = load 'table_testForEachSameOriginColumn3' as (a0:int, a1:int, a2:int);");
+        pigServer.registerQuery("b = group a by a0;");
+        pigServer.registerQuery("c = foreach b generate flatten(a.(a1,a2)) as (b0, b1), flatten(a.(a1,a2)) as (b2, b3);");
+        pigServer.registerQuery("d = filter c by b0>b2;");
+        
+        Iterator<Tuple> iter = pigServer.openIterator("d");
+        
+        Tuple t = iter.next();
+        assertTrue(t.toString().contains("(2,3,1,2)"));
+        assertFalse(iter.hasNext());
+    }
 }