You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/03/19 07:47:53 UTC
svn commit: r925105 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
test/org/apache/pig/test/TestPushUpFilter.java
Author: daijy
Date: Fri Mar 19 06:47:53 2010
New Revision: 925105
URL: http://svn.apache.org/viewvc?rev=925105&view=rev
Log:
PIG-1289: PIG Join fails while doing a filter on joined data
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925105&r1=925104&r2=925105&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Mar 19 06:47:53 2010
@@ -157,6 +157,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1289: PIG Join fails while doing a filter on joined data (daijy)
+
PIG-1266: Show spill count on the pig console at the end of the job (sriranjan
via rding)
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java?rev=925105&r1=925104&r2=925105&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java Fri Mar 19 06:47:53 2010
@@ -31,6 +31,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.LOCast;
import org.apache.pig.impl.logicalLayer.LOCogroup;
import org.apache.pig.impl.logicalLayer.LOCross;
+import org.apache.pig.impl.logicalLayer.LOIsNull;
import org.apache.pig.impl.logicalLayer.LOJoin;
import org.apache.pig.impl.logicalLayer.LOFilter;
import org.apache.pig.impl.logicalLayer.LOForEach;
@@ -231,8 +232,32 @@ public class PushUpFilter extends Logica
return false;
}
}
- mPushBefore = true;
+
mPushBeforeInput = grandParentIndexes.iterator().next();
+
+ if (predecessor instanceof LOJoin) {
+ boolean otherBranchContainOuter = false;
+ boolean sawInner = false;
+ for (int i=0;i<=mPlan.getSuccessors(predecessor).size();i++) {
+ // We do not push filter if any other branch is outer
+ // See PIG-1289
+ // Also in LOJoin, innerFlag==true indicate that branch is the outer join side
+ // which has the exact opposite semantics
+ // If all innerFlag is true, that implies a regular join
+ if (i!=mPushBeforeInput && ((LOJoin)predecessor).getInnerFlags()[i]) {
+ otherBranchContainOuter = true;
+ }
+ if (((LOJoin)predecessor).getInnerFlags()[i]==false) {
+ sawInner = true;
+ }
+ }
+ if (otherBranchContainOuter && sawInner) {
+ mPushBeforeInput = -1;
+ return false;
+ }
+ }
+
+ mPushBefore = true;
return true;
} else if (predecessor instanceof LOForEach) {
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java?rev=925105&r1=925104&r2=925105&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Fri Mar 19 06:47:53 2010
@@ -1068,6 +1068,24 @@ public class TestPushUpFilter extends ju
assertTrue(pushUpFilter.getPushBefore() == false);
assertTrue(pushUpFilter.getPushBeforeInput() == -1);
}
+
+ @Test
+ public void testOutJoin() throws Exception {
+ planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
+ planTester.buildPlan("B = load 'anotherfile' as (name);");
+ planTester.buildPlan("C = join A by name LEFT OUTER, B by name;");
+ LogicalPlan lp = planTester.buildPlan("D = filter C by B::name is null;");
+
+ planTester.setPlan(lp);
+ planTester.setProjectionMap(lp);
+
+ PushUpFilter pushUpFilter = new PushUpFilter(lp);
+
+ assertTrue(!pushUpFilter.check(lp.getLeaves()));
+ assertTrue(pushUpFilter.getSwap() == false);
+ assertTrue(pushUpFilter.getPushBefore() == false);
+ assertTrue(pushUpFilter.getPushBeforeInput() == -1);
+ }
}