You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/07/26 12:29:47 UTC
svn commit: r1754121 - in /pig/branches/branch-0.16: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
test/org/apache/pig/test/
Author: rohini
Date: Tue Jul 26 12:29:47 2016
New Revision: 1754121
URL: http://svn.apache.org/viewvc?rev=1754121&view=rev
Log:
PIG-4961: CROSS followed by LIMIT inside nested foreach drop data from result (rohini)
Modified:
pig/branches/branch-0.16/CHANGES.txt
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCross.java
pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
pig/branches/branch-0.16/test/org/apache/pig/test/TestForEachNestedPlanLocal.java
Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1754121&r1=1754120&r2=1754121&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Tue Jul 26 12:29:47 2016
@@ -30,6 +30,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4961: CROSS followed by LIMIT inside nested foreach drop data from result (rohini)
+
PIG-4960: Split followed by order by/skewed join is skewed in Tez (rohini)
PIG-4947: LOAD with HBaseStorage using a mix of pure wildcards and prefixed wildcards results in
Modified: pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCross.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCross.java?rev=1754121&r1=1754120&r2=1754121&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCross.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCross.java Tue Jul 26 12:29:47 2016
@@ -256,4 +256,9 @@ public class POCross extends PhysicalOpe
data = null;
}
+ @Override
+ public void reset() {
+ clearMemory();
+ }
+
}
Modified: pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java?rev=1754121&r1=1754120&r2=1754121&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java Tue Jul 26 12:29:47 2016
@@ -738,9 +738,12 @@ public class POForEach extends PhysicalO
opsToBeReset.add(sort);
}
- /* (non-Javadoc)
- * @see org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor#visitProject(org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject)
- */
+ @Override
+ public void visitCross(POCross c) throws VisitorException {
+ // FIXME: add only if limit is present
+ opsToBeReset.add(c);
+ }
+
@Override
public void visitProject(POProject proj) throws VisitorException {
if(proj instanceof PORelationToExprProject) {
Modified: pig/branches/branch-0.16/test/org/apache/pig/test/TestForEachNestedPlanLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/TestForEachNestedPlanLocal.java?rev=1754121&r1=1754120&r2=1754121&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/TestForEachNestedPlanLocal.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/TestForEachNestedPlanLocal.java Tue Jul 26 12:29:47 2016
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Random;
import org.apache.pig.PigServer;
+import org.apache.pig.builtin.mock.Storage;
import org.apache.pig.data.Tuple;
import org.apache.pig.test.utils.TestHelper;
import org.junit.Test;
@@ -111,6 +112,31 @@ public class TestForEachNestedPlanLocal
}
@Test
+ public void testNestedCrossTwoRelationsLimit() throws Exception {
+ Storage.Data data = Storage.resetData(pig);
+ data.set("input",
+ Storage.tuple(Storage.bag(Storage.tuple(1, 1), Storage.tuple(1, 2)), Storage.bag(Storage.tuple(1, 3), Storage.tuple(1, 4))),
+ Storage.tuple(Storage.bag(Storage.tuple(2, 1), Storage.tuple(2, 2)), Storage.bag(Storage.tuple(2, 3))),
+ Storage.tuple(Storage.bag(Storage.tuple(3, 1)), Storage.bag(Storage.tuple(3, 2))));
+
+ pig.setBatchOn();
+ pig.registerQuery("A = load 'input' using mock.Storage() as (bag1:bag{tup1:tuple(f1:int, f2:int)}, bag2:bag{tup2:tuple(f3:int, f4:int)});");
+ pig.registerQuery("B = foreach A {"
+ + "crossed = cross bag1, bag2;"
+ + "filtered = filter crossed by f1 == f3;"
+ + "lmt = limit filtered 1;"
+ + "generate FLATTEN(lmt);" + "}");
+ pig.registerQuery("store B into 'output' using mock.Storage();");
+
+ pig.executeBatch();
+
+ List<Tuple> actualResults = data.get("output");
+ List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
+ new String[] {"(1, 1, 1, 3)", "(2, 1, 2, 3)", "(3, 1, 3, 2)"});
+ Util.checkQueryOutputs(actualResults.iterator(), expectedResults);
+ }
+
+ @Test
public void testNestedCrossTwoRelationsComplex() throws Exception {
File[] tmpFiles = generateDataSetFilesForNestedCross();
List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStringAsByteArray(new String[] {