You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/03/21 04:58:25 UTC
svn commit: r1579873 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
Author: daijy
Date: Fri Mar 21 03:58:25 2014
New Revision: 1579873
URL: http://svn.apache.org/r1579873
Log:
PIG-3826: Outer join with PushDownForEachFlatten generates wrong result
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1579873&r1=1579872&r2=1579873&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Mar 21 03:58:25 2014
@@ -257,6 +257,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3826: Outer join with PushDownForEachFlatten generates wrong result (daijy)
+
PIG-3820: TestAvroStorage fail on some OS (daijy)
PIG-3818: PIG-2499 is accidently reverted (daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java?rev=1579873&r1=1579872&r2=1579873&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java Fri Mar 21 03:58:25 2014
@@ -159,6 +159,11 @@ public class PushDownForEachFlatten exte
for( int i = 0; i < preds.size(); i++ ) {
Operator op = preds.get( i );
if( op == foreach ) {
+ // Don't optimize if the flattened side is outer side of an outer join
+ // See PIG-3826
+ if (join.getInnerFlags()[i]==false) {
+ return false;
+ }
Collection<LogicalExpressionPlan> exprs = join.getJoinPlan( i );
for( LogicalExpressionPlan expr : exprs ) {
List<ProjectExpression> projs = getProjectExpressions( expr );
Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1579873&r1=1579872&r2=1579873&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Fri Mar 21 03:58:25 2014
@@ -1215,5 +1215,21 @@ public class TestNewPlanPushDownForeachF
Assert.assertTrue( sort instanceof LOSort );
}
+
+ @Test
+ // See PIG-3826
+ public void testOuterJoin() throws Exception {
+ String query = "A = load 'A.txt' as (id:chararray, value:double);" +
+ "B = load 'B.txt' as (id:chararray, name:chararray);" +
+ "t1 = group A by id;" +
+ "t2 = foreach t1 { r1 = filter $1 by (value>1); r2 = limit r1 1; generate group as id, FLATTEN(r2.value) as value; }" +
+ "t3 = join B by id LEFT OUTER, t2 by id;" +
+ "store t3 into 'output';";
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
+
+ Operator store = newLogicalPlan.getSinks().get( 0 );
+ Operator join = newLogicalPlan.getPredecessors(store).get(0);
+ Assert.assertTrue( join instanceof LOJoin );
+ }
}