You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/06/28 20:10:52 UTC

svn commit: r1140767 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/PigServer.java src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java test/org/apache/pig/test/TestFilterSimplification.java

Author: thejas
Date: Tue Jun 28 18:10:50 2011
New Revision: 1140767

URL: http://svn.apache.org/viewvc?rev=1140767&view=rev
Log:
PIG-2144: ClassCastException when using IsEmpty(DIFF())

Modified:
    pig/branches/branch-0.9/CHANGES.txt
    pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
    pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java

Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Jun 28 18:10:50 2011
@@ -192,6 +192,8 @@ PIG-1696: Performance: Use System.arrayc
 
 BUG FIXES
 
+PIG-2144: ClassCastException when using IsEmpty(DIFF()) (thejas)
+
 PIG-2139: LogicalExpressionSimplifier optimizer rule should check if udf is
  deterministic while checking if they are equal (thejas)
 

Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Tue Jun 28 18:10:50 2011
@@ -1046,20 +1046,6 @@ public class PigServer {
                 return;
             }
             PhysicalPlan pp = compilePp();
-
-            HashSet<String> optimizerRules = null;
-            try {
-                optimizerRules = (HashSet<String>) ObjectSerializer.deserialize(
-                        pigContext.getProperties().getProperty("pig.optimizer.rules") );
-            } catch (IOException ioe) {
-                int errCode = 2110;
-                String msg = "Unable to deserialize optimizer rules.";
-                throw new FrontendException(msg, errCode, PigException.BUG, ioe);
-            }
-
-            LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer( currDAG.lp, 3, optimizerRules );
-            optimizer.optimize();
-
             currDAG.lp.explain(lps, format, verbose);
 
             pp.explain(pps, format, verbose);

Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java Tue Jun 28 18:10:50 2011
@@ -265,8 +265,16 @@ class NOTConversionVisitor extends Logic
                 if (seen.add(suc)) {
                     if (suc instanceof NotExpression)
                         ((NOTConversionVisitor) visitor).flip();
-                    Collection<Operator> newSuccessors = Utils.mergeCollection(plan.getSuccessors(suc), plan.getSoftLinkSuccessors(suc));
-                    depthFirst(suc, newSuccessors, seen, visitor);
+                    if(suc instanceof AndExpression
+                            || suc instanceof NotExpression
+                            || suc instanceof OrExpression
+                    ){
+                        //visit successors of suc only if they are the boolean operators
+                        // the NOT conversion should be propagated only for 
+                        // their successors 
+                        Collection<Operator> newSuccessors = Utils.mergeCollection(plan.getSuccessors(suc), plan.getSoftLinkSuccessors(suc));
+                        depthFirst(suc, newSuccessors, seen, visitor);
+                    }
                     suc.accept(visitor);
                     if (suc instanceof NotExpression)
                         ((NOTConversionVisitor) visitor).flip();

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java Tue Jun 28 18:10:50 2011
@@ -21,6 +21,7 @@ package org.apache.pig.test;
 import java.util.*;
 
 import org.apache.pig.ExecType;
+import org.apache.pig.FilterFunc;
 import org.apache.pig.PigServer;
 import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
 import org.apache.pig.newplan.logical.relational.LOFilter;
@@ -31,6 +32,8 @@ import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.OperatorPlan;
 import org.apache.pig.newplan.optimizer.PlanOptimizer;
 import org.apache.pig.newplan.optimizer.Rule;
+import org.apache.pig.test.TestPruneColumn.MyFilterFunc;
+import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 
 import junit.framework.TestCase;
@@ -766,7 +769,33 @@ public class TestFilterSimplification ex
         );
 
     }
+    
+    static public class MyFilterFunc extends FilterFunc {
+        
+        @Override
+        public Boolean exec(Tuple input) {
+            return true;
+        }
+    }
+
+    //PIG-2144
+    @Test
+    public void testNotConversionUdfArg() throws Exception{
+        //udf arg should not be changed
+        String query = "b = filter (load 'd.txt' as (a0, a1)) by " +
+        " NOT IsEmpty( " +  MyFilterFunc.class.getName() + "(a0,a1));" + 
+        "store b into 'empty';";
+        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);;
 
+        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+        optimizer.optimize();
+        
+        //expected plan is same as original plan
+        LogicalPlan expected = Util.buildLp(pigServer, query);;
+
+        assertTrue(expected.isEqual(newLogicalPlan));
+    }
+    
     public class MyPlanOptimizer extends LogicalPlanOptimizer {
 
         protected MyPlanOptimizer(OperatorPlan p, int iterations) {