You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/07/22 03:10:42 UTC
svn commit: r966470 - in /hadoop/pig/branches/branch-0.7: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
test/org/apache/pig/test/TestPushUpFilter.java
Author: daijy
Date: Thu Jul 22 01:10:42 2010
New Revision: 966470
URL: http://svn.apache.org/viewvc?rev=966470&view=rev
Log:
PIG-1507: Full outer join fails while doing a filter on joined data
Modified:
hadoop/pig/branches/branch-0.7/CHANGES.txt
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestPushUpFilter.java
Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=966470&r1=966469&r2=966470&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Thu Jul 22 01:10:42 2010
@@ -198,6 +198,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1507: Full outer join fails while doing a filter on joined data (daijy)
+
PIG-1493: Column Pruner throw exception "inconsistent pruning" (daijy)
PIG-1490: Make Pig storers work with remote HDFS in secure mode (rding)
Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java?rev=966470&r1=966469&r2=966470&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java (original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java Thu Jul 22 01:10:42 2010
@@ -244,6 +244,7 @@ public class PushUpFilter extends Logica
// Also in LOJoin, innerFlag==true indicate that branch is the outer join side
// which has the exact opposite semantics
// If all innerFlag is true, that implies a regular join
+ // If all innerFlag is false, means a outer join, in this case, we can not push up filter for any path (See PIG-1507)
if (i!=mPushBeforeInput && ((LOJoin)predecessor).getInnerFlags()[i]) {
otherBranchContainOuter = true;
}
@@ -251,7 +252,13 @@ public class PushUpFilter extends Logica
sawInner = true;
}
}
- if (otherBranchContainOuter && sawInner) {
+ if (!otherBranchContainOuter && ((LOJoin)predecessor).getInnerFlags()[mPushBeforeInput]==false) // all innerFlag is false, implies an outer join
+ {
+ mPushBeforeInput = -1;
+ return false;
+ }
+ if (otherBranchContainOuter && sawInner) // If it is not a regular join and the path we push is on inner side
+ {
mPushBeforeInput = -1;
return false;
}
Modified: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestPushUpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestPushUpFilter.java?rev=966470&r1=966469&r2=966470&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestPushUpFilter.java (original)
+++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestPushUpFilter.java Thu Jul 22 01:10:42 2010
@@ -1069,6 +1069,7 @@ public class TestPushUpFilter extends ju
assertTrue(pushUpFilter.getPushBeforeInput() == -1);
}
+ // See PIG-1289
@Test
public void testOutJoin() throws Exception {
planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
@@ -1086,6 +1087,25 @@ public class TestPushUpFilter extends ju
assertTrue(pushUpFilter.getPushBefore() == false);
assertTrue(pushUpFilter.getPushBeforeInput() == -1);
}
+
+ // See PIG-1507
+ @Test
+ public void testFullOutJoin() throws Exception {
+ planTester.buildPlan("A = load 'myfile' as (d1:int);");
+ planTester.buildPlan("B = load 'anotherfile' as (d2:int);");
+ planTester.buildPlan("c = join A by d1 full outer, B by d2;");
+ LogicalPlan lp = planTester.buildPlan("d = filter c by d2 is null;");
+
+ planTester.setPlan(lp);
+ planTester.setProjectionMap(lp);
+
+ PushUpFilter pushUpFilter = new PushUpFilter(lp);
+
+ assertTrue(!pushUpFilter.check(lp.getLeaves()));
+ assertTrue(pushUpFilter.getSwap() == false);
+ assertTrue(pushUpFilter.getPushBefore() == false);
+ assertTrue(pushUpFilter.getPushBeforeInput() == -1);
+ }
}