You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2010/07/30 21:14:23 UTC
svn commit: r980930 - in /hadoop/pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
test/org/apache/pig/test/ test/org/apache/pig/test/data/GoldenFiles/
Author: thejas
Date: Fri Jul 30 19:14:22 2010
New Revision: 980930
URL: http://svn.apache.org/viewvc?rev=980930&view=rev
Log:
PIG-1521: explain plan does not show correct Physical operator in MR plan when POSortedDistinct, POPackageLite are used (thejas)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackageLite.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSortedDistinct.java
hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=980930&r1=980929&r2=980930&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 30 19:14:22 2010
@@ -110,6 +110,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1521: explain plan does not show correct Physical operator in MR plan when POSortedDistinct, POPackageLite are used (thejas)
+
PIG-1513: Pig doesn't handle empty input directory (rding)
PIG-1500: guava.jar should be removed from the lib folder (niraj via rding)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackageLite.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackageLite.java?rev=980930&r1=980929&r2=980930&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackageLite.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackageLite.java Fri Jul 30 19:14:22 2010
@@ -195,5 +195,12 @@ public class POPackageLite extends POPac
return retTuple;
}
+ @Override
+ public String name() {
+ return getAliasString() + "PackageLite" + "["
+ + DataType.findTypeName(resultType) + "]" + "{"
+ + DataType.findTypeName(keyType) + "}" + " - "
+ + mKey.toString();
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSortedDistinct.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSortedDistinct.java?rev=980930&r1=980929&r2=980930&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSortedDistinct.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSortedDistinct.java Fri Jul 30 19:14:22 2010
@@ -23,6 +23,7 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.plan.OperatorKey;
@@ -83,4 +84,11 @@ public class POSortedDistinct extends PO
return in;
}
}
+
+ @Override
+ public String name() {
+ return getAliasString() + "POSortedDistinct" + "["
+ + DataType.findTypeName(resultType) + "]" + " - "
+ + mKey.toString();
+ }
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=980930&r1=980929&r2=980930&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Fri Jul 30 19:14:22 2010
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.Random;
@@ -810,6 +811,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC16.gld");
}
+
public void testLimit() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -991,6 +993,33 @@ public class TestMRCompiler extends juni
assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
}
+ /**
+ * Test that POSortedDistinct gets printed as POSortedDistinct
+ * @throws Exception
+ */
+ @Test
+ public void testSortedDistinctInForeach() throws Exception {
+ PhysicalPlan php = new PhysicalPlan();
+ PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
+ php.merge(grpChain1);
+
+ List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
+ PhysicalPlan inplan = new PhysicalPlan();
+ PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()),
+ -1, null);
+ inplan.addAsLeaf(op1);
+ inputs.add(inplan);
+ List<Boolean> toFlattens = new ArrayList<Boolean>();
+ toFlattens.add(false);
+ POForEach pofe = new POForEach(new OperatorKey("", r.nextLong()), 1,
+ inputs, toFlattens);
+
+ php.addAsLeaf(pofe);
+ POStore st = GenPhyOp.topStoreOp();
+ php.addAsLeaf(st);
+ run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC19.gld");
+ }
+
private void run(PhysicalPlan pp, String expectedFile) throws Exception {
String compiledPlan, goldenPlan = null;
int MAX_SIZE = 100000;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=980930&r1=980929&r2=980930&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Fri Jul 30 19:14:22 2010
@@ -43,7 +43,7 @@ MapReduce(1,GFCross) - -156:
| | |
| | Project[bag][1] - -149
| |
- | |---Package[tuple]{tuple} - -148
+ | |---PackageLite[tuple]{tuple} - -148
| Local Rearrange[tuple]{tuple}(false) - -147
| | |
| | Project[tuple][*] - -146
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld?rev=980930&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld Fri Jul 30 19:14:22 2010
@@ -0,0 +1,11 @@
+MapReduce(1) - -181:
+| Store(DummyFil:DummyLdr) - -3088212343542276753
+| |
+| |---New For Each(false)[tuple] - --274910157919533456
+| | |
+| | POSortedDistinct[tuple] - --881122551222328650
+| |
+| |---Package[tuple]{Unknown} - -49912403876500969
+| Local Rearrange[tuple]{Unknown}(false) - --790930717481674068
+| |
+| |---Load(DummyFil:DummyLdr) - -4086340370247387675
\ No newline at end of file