You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/06/28 20:10:52 UTC
svn commit: r1140767 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/PigServer.java
src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java
test/org/apache/pig/test/TestFilterSimplification.java
Author: thejas
Date: Tue Jun 28 18:10:50 2011
New Revision: 1140767
URL: http://svn.apache.org/viewvc?rev=1140767&view=rev
Log:
PIG-2144: ClassCastException when using IsEmpty(DIFF())
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Jun 28 18:10:50 2011
@@ -192,6 +192,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2144: ClassCastException when using IsEmpty(DIFF()) (thejas)
+
PIG-2139: LogicalExpressionSimplifier optimizer rule should check if udf is
deterministic while checking if they are equal (thejas)
Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Tue Jun 28 18:10:50 2011
@@ -1046,20 +1046,6 @@ public class PigServer {
return;
}
PhysicalPlan pp = compilePp();
-
- HashSet<String> optimizerRules = null;
- try {
- optimizerRules = (HashSet<String>) ObjectSerializer.deserialize(
- pigContext.getProperties().getProperty("pig.optimizer.rules") );
- } catch (IOException ioe) {
- int errCode = 2110;
- String msg = "Unable to deserialize optimizer rules.";
- throw new FrontendException(msg, errCode, PigException.BUG, ioe);
- }
-
- LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer( currDAG.lp, 3, optimizerRules );
- optimizer.optimize();
-
currDAG.lp.explain(lps, format, verbose);
pp.explain(pps, format, verbose);
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/NotConversionVisitor.java Tue Jun 28 18:10:50 2011
@@ -265,8 +265,16 @@ class NOTConversionVisitor extends Logic
if (seen.add(suc)) {
if (suc instanceof NotExpression)
((NOTConversionVisitor) visitor).flip();
- Collection<Operator> newSuccessors = Utils.mergeCollection(plan.getSuccessors(suc), plan.getSoftLinkSuccessors(suc));
- depthFirst(suc, newSuccessors, seen, visitor);
+ if(suc instanceof AndExpression
+ || suc instanceof NotExpression
+ || suc instanceof OrExpression
+ ){
+ //visit successors of suc only if they are the boolean operators
+ // the NOT conversion should be propagated only for
+ // their successors
+ Collection<Operator> newSuccessors = Utils.mergeCollection(plan.getSuccessors(suc), plan.getSoftLinkSuccessors(suc));
+ depthFirst(suc, newSuccessors, seen, visitor);
+ }
suc.accept(visitor);
if (suc instanceof NotExpression)
((NOTConversionVisitor) visitor).flip();
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java?rev=1140767&r1=1140766&r2=1140767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestFilterSimplification.java Tue Jun 28 18:10:50 2011
@@ -21,6 +21,7 @@ package org.apache.pig.test;
import java.util.*;
import org.apache.pig.ExecType;
+import org.apache.pig.FilterFunc;
import org.apache.pig.PigServer;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOFilter;
@@ -31,6 +32,8 @@ import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
import org.apache.pig.newplan.optimizer.Rule;
+import org.apache.pig.test.TestPruneColumn.MyFilterFunc;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import junit.framework.TestCase;
@@ -766,7 +769,33 @@ public class TestFilterSimplification ex
);
}
+
+ static public class MyFilterFunc extends FilterFunc {
+
+ @Override
+ public Boolean exec(Tuple input) {
+ return true;
+ }
+ }
+
+ //PIG-2144
+ @Test
+ public void testNotConversionUdfArg() throws Exception{
+ //udf arg should not be changed
+ String query = "b = filter (load 'd.txt' as (a0, a1)) by " +
+ " NOT IsEmpty( " + MyFilterFunc.class.getName() + "(a0,a1));" +
+ "store b into 'empty';";
+ LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);;
+ PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+ optimizer.optimize();
+
+ //expected plan is same as original plan
+ LogicalPlan expected = Util.buildLp(pigServer, query);;
+
+ assertTrue(expected.isEqual(newLogicalPlan));
+ }
+
public class MyPlanOptimizer extends LogicalPlanOptimizer {
protected MyPlanOptimizer(OperatorPlan p, int iterations) {