You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/04/06 12:54:14 UTC
svn commit: r1585283 [3/3] - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/fetch/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/
src/org/ap...
Modified: pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java (original)
+++ pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java Sun Apr 6 10:54:13 2014
@@ -19,69 +19,59 @@
package org.apache.pig.pen;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.Map;
-import java.util.Collection;
import java.util.List;
+import java.util.Map;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMultiQueryPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GreaterThanExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LessThanExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.EqualToExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GTOrEqualToExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GreaterThanExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LTOrEqualToExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.EqualToExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LessThanExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.NotEqualToExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORegexp;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POIsNull;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POAnd;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POOr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONot;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POIsNull;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONegative;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONot;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POOr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORegexp;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POJoinPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.LitePackager;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartitionRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POOptimizedForEach;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPreCombinerLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.PlanWalker;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.IdentityHashSet;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
-import org.apache.pig.pen.util.LineageTracer;
-import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.pen.util.LineageTracer;
/**
* The class used to (re)attach illustrators to physical operators
@@ -206,20 +196,7 @@ public class IllustratorAttacher extends
@Override
public void visitPackage(POPackage pkg) throws VisitorException{
- if (!(pkg instanceof POPackageLite) && pkg.isDistinct())
- setIllustrator(pkg, 1);
- else
- setIllustrator(pkg, null);
- }
-
- @Override
- public void visitCombinerPackage(POCombinerPackage pkg) throws VisitorException{
- setIllustrator(pkg);
- }
-
- @Override
- public void visitMultiQueryPackage(POMultiQueryPackage pkg) throws VisitorException{
- setIllustrator(pkg);
+ setIllustrator(pkg, pkg.numberOfEquivalenceClasses());
}
@Override
@@ -230,10 +207,9 @@ public class IllustratorAttacher extends
for (PhysicalPlan innerPlan : innerPlans)
innerPlanAttach(nfe, innerPlan);
List<PhysicalOperator> preds = mPlan.getPredecessors(nfe);
- if (preds != null && preds.size() == 1 &&
- preds.get(0) instanceof POPackage &&
- !(preds.get(0) instanceof POPackageLite) &&
- ((POPackage) preds.get(0)).isDistinct()) {
+ if (preds != null && preds.size() == 1
+ && preds.get(0) instanceof POPackage
+ && ((POPackage) preds.get(0)).getPkgr().isDistinct()) {
// equivalence class of POPackage for DISTINCT needs to be used
//instead of the succeeding POForEach's equivalence class
setIllustrator(nfe, preds.get(0).getIllustrator().getEquivalenceClasses());
@@ -391,14 +367,6 @@ public class IllustratorAttacher extends
}
@Override
- public void visitJoinPackage(POJoinPackage joinPackage) throws VisitorException{
- if (revisit && joinPackage.getIllustrator() != null)
- return;
- setIllustrator(joinPackage);
- joinPackage.getForEach().setIllustrator(joinPackage.getIllustrator());
- }
-
- @Override
public void visitCast(POCast cast) {
}
Modified: pig/trunk/src/org/apache/pig/pen/PhysicalPlanResetter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/PhysicalPlanResetter.java?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/PhysicalPlanResetter.java (original)
+++ pig/trunk/src/org/apache/pig/pen/PhysicalPlanResetter.java Sun Apr 6 10:54:13 2014
@@ -17,25 +17,12 @@
*/
package org.apache.pig.pen;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.PigException;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POJoinPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
-import org.apache.pig.impl.plan.optimizer.OptimizerException;
-import org.apache.pig.impl.util.Pair;
/**
* This visitor visits the physical plan and resets it for next MRCompilation
@@ -48,6 +35,6 @@ public class PhysicalPlanResetter extend
@Override
public void visitPackage(POPackage pkg) throws VisitorException {
- pkg.setKeyInfo(null);
+ pkg.getPkgr().setKeyInfo(null);
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java Sun Apr 6 10:54:13 2014
@@ -45,7 +45,7 @@ public class TestExampleGenerator {
static int MAX = 100;
static String A, B;
static File fileA, fileB;
-
+
@BeforeClass
public static void oneTimeSetup() throws Exception {
pigContext.connect();
@@ -55,7 +55,7 @@ public class TestExampleGenerator {
writeData(fileA);
writeData(fileB);
-
+
fileA.deleteOnExit();
fileB.deleteOnExit();
@@ -122,7 +122,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testFilter3() throws Exception {
@@ -144,7 +144,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testForeach() throws ExecException, IOException {
PigServer pigServer = new PigServer(pigContext);
@@ -157,7 +157,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
//see PIG-2170
@Test
public void testForeachBinCondWithBooleanExp() throws ExecException, IOException {
@@ -171,7 +171,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testForeachWithTypeCastCounter() throws ExecException, IOException {
PigServer pigServer = new PigServer(pigContext);
@@ -245,7 +245,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testGroup2() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -270,7 +270,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testFilterUnion() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -283,7 +283,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testForEachNestedBlock() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -307,7 +307,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testUnion() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -328,7 +328,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testCross() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -339,7 +339,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testLimit() throws Exception {
PigServer pigServer = new PigServer(pigContext);
@@ -349,7 +349,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
//see PIG-2275
@Test
public void testFilterWithIsNull() throws ExecException, IOException {
@@ -363,7 +363,7 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
-
+
@Test
public void testFilterWithUDF() throws ExecException, IOException {
PigServer pigServer = new PigServer(pigContext);
@@ -383,7 +383,7 @@ public class TestExampleGenerator {
File out = File.createTempFile("testFilterGroupCountStoreOutput", "");
out.deleteOnExit();
out.delete();
-
+
PigServer pigServer = new PigServer(pigContext);
pigServer.setBatchOn();
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
@@ -392,16 +392,26 @@ public class TestExampleGenerator {
pigServer.registerQuery("D = foreach C generate group as x, COUNT(B) as the_count;");
pigServer.registerQuery("store D into '" + Util.encodeEscape(out.getAbsolutePath()) + "';");
Map<Operator, DataBag> derivedData = pigServer.getExamples(null);
-
+
assertNotNull(derivedData);
}
-
+
@Test
public void testLoaderWithContext() throws Exception {
PigServer pigServer = new PigServer(pigContext);
pigServer.registerQuery("A = load " + A.toString() + " using " + UDFContextTestLoaderWithSignature.class.getName() + "('a') as (x, y);");
Map<Operator, DataBag> derivedData = pigServer.getExamples("A");
-
+
+ assertNotNull(derivedData);
+ }
+
+ @Test
+ public void testOrderBy() throws Exception {
+ PigServer pigServer = new PigServer(pigContext);
+ pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
+ pigServer.registerQuery("B = order A by x;");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
+
assertNotNull(derivedData);
}
Modified: pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobSubmission.java?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJobSubmission.java Sun Apr 6 10:54:13 2014
@@ -100,7 +100,7 @@ public class TestJobSubmission {
if (Util.WINDOWS) {
inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
golDir="/"+FileLocalizer.parseCygPath(golDir, FileLocalizer.STYLE_WINDOWS);
- }
+ }
}
@After
@@ -112,329 +112,6 @@ public class TestJobSubmission {
cluster.shutDown();
}
-/* private void generateInput(int numTuples) throws ExecException{
-
- DataBag inpDb = GenRandomData.genRandSmallTupDataBag(r, numTuples, 1000);
-
- POProject proj = new POProject(new OperatorKey("", r.nextLong()));
- Tuple t = new DefaultTuple();
- t.append(inpDb);
- proj.attachInput(t);
- proj.setColumn(0);
- proj.setOverloaded(true);
- proj.setResultType(DataType.TUPLE);
-
- List<PhysicalOperator> inps = new ArrayList<PhysicalOperator>();
- inps.add(proj);
-
- POStore str = new POStore(new OperatorKey("", r.nextLong()));
- str.setInputs(inps);
-
- FileSpec fSpec = new FileSpec(ldFile, new FuncSpec(PigStorage.class.getName()));
-
- str.setSFile(fSpec);
- str.setPc(pc);
- str.store();
- }
-
- private void setUp1(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = ldFile;
- stFile = "jsTst1";
- grpName = "jobSubTst1";
-
- if(gen){
- generateInput(100);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- php.add(ld);
- php.add(st);
- php.connect(ld, st);
- }
-
-// @Test
- public void testCompile1() throws Exception {
- boolean gen = false;
-
- setUp1(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec fSpecAct = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp2(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst2.txt";
- expFile = ldFile;
- stFile = "jsTst2";
- grpName = "jobSubTst2";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- php.add(ld);
- php.add(st);
- php.connect(ld, st);
- }
-
-// @Test
- public void testCompile2() throws Exception {
- boolean gen = false;
-
- setUp2(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp3(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = "file:" + golDir + "jsTst3";
- stFile = "jsTst3";
- grpName = "jobSubTst3";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- int[] flds = {0,1};
- Tuple sample = new DefaultTuple();
- sample.append(new String("S"));
- sample.append(new Integer("10"));
-
- POForEach fe = GenPhyOp.topForEachOPWithPlan(flds , sample);
-
- POFilter fl = GenPhyOp.topFilterOpWithProj(1, 500, GenPhyOp.LT);
-
- php.add(ld);
- php.add(fe);
- php.connect(ld, fe);
-
- php.add(fl);
- php.connect(fe, fl);
-
- php.add(st);
- php.connect(fl, st);
- }
-
-// @Test
- public void testCompile3() throws Exception {
- boolean gen = false;
-
- setUp3(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp4(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = "file:" + golDir + "jsTst4";
- stFile = "jsTst4";
- grpName = "jobSubTst4";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- POSplit spl = GenPhyOp.topSplitOp();
- POFilter fl1 = GenPhyOp.topFilterOpWithProjWithCast(1, 200, GenPhyOp.LT);
- POFilter fl2 = GenPhyOp.topFilterOpWithProjWithCast(1, 800, GenPhyOp.GT);
-
- POUnion un = GenPhyOp.topUnionOp();
-
- php.add(ld);
- php.add(spl);
- php.connect(ld, spl);
-
- php.add(fl1);
- php.connect(spl, fl1);
-
- php.add(fl2);
- php.connect(spl, fl2);
-
- php.add(un);
- php.connect(fl1, un);
- php.connect(fl2, un);
-
- php.add(st);
- php.connect(un, st);
- }
-
-// @Test
- public void testCompile4() throws Exception {
- boolean gen = false;
-
- setUp4(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
-
- }
-
- private void setUp5(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst5.txt";
- expFile = ldFile;
- stFile = "jsTst5";
- grpName = "jobSubTst5";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- Tuple sample = new DefaultTuple();
- sample.append("S");
- sample.append(1);
- POLocalRearrange lr = GenPhyOp.topLocalRearrangeOPWithPlan(0, 1, sample);
-
- POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
-
- POPackage pk = GenPhyOp.topPackageOp();
- pk.setKeyType(DataType.INTEGER);
- pk.setNumInps(1);
- boolean[] inner = {false};
- pk.setInner(inner);
-
- POForEach fe = GenPhyOp.topForEachOPWithPlan(1);
-
- php.add(ld);
- php.add(lr);
- php.connect(ld, lr);
-
- php.add(gr);
- php.connect(lr, gr);
-
- php.add(pk);
- php.connect(gr, pk);
-
- php.add(fe);
- php.connect(pk, fe);
-
- php.add(st);
- php.connect(fe, st);
- }
-
- @Test
- public void testCompile5() throws Exception {
- boolean gen = false;
-
- setUp5(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
-
- }*/
-
@Test
public void testJobControlCompilerErr() throws Exception {
String query = "a = load 'input';" + "b = order a by $0;" + "store b into 'output';";
@@ -443,17 +120,17 @@ public class TestJobSubmission {
POStore store = GenPhyOp.dummyPigStorageOp();
pp.addAsLeaf(store);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
-
+
for(MapReduceOper mro: mrPlan.getLeaves()) {
if(mro.reducePlan != null) {
PhysicalOperator po = mro.reducePlan.getRoots().get(0);
- if(po instanceof POPackage) {
- ((POPackage)po).setKeyType(DataType.BAG);
+ if (po instanceof POPackage) {
+ ((POPackage) po).getPkgr().setKeyType(DataType.BAG);
mro.setGlobalSort(true);
}
}
}
-
+
ConfigurationValidator.validatePigProperties(pc.getProperties());
Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
JobControlCompiler jcc = new JobControlCompiler(pc, conf);
@@ -517,9 +194,9 @@ public class TestJobSubmission {
// default_parallel is considered only at runtime, so here we only test requested parallel
// more thorough tests can be found in TestNumberOfReducers.java
String query = "a = load 'input';" +
- "b = load 'input';" +
- "c = join a by $0, b by $0 using 'skewed' parallel 100;" +
- "store c into 'output';";
+ "b = load 'input';" +
+ "c = join a by $0, b by $0 using 'skewed' parallel 100;" +
+ "store c into 'output';";
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PhysicalPlan pp = Util.buildPp(ps, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
@@ -551,8 +228,8 @@ public class TestJobSubmission {
util.startMiniHBaseCluster(1, 1);
String query = "a = load '/passwd';" +
- "b = group a by $0;" +
- "store b into 'output';";
+ "b = group a by $0;" +
+ "store b into 'output';";
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PhysicalPlan pp = Util.buildPp(ps, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
@@ -620,13 +297,13 @@ public class TestJobSubmission {
@Test
public void testReducerNumEstimationForOrderBy() throws Exception{
- // use the estimation
+ // use the estimation
pc.getProperties().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
pc.getProperties().setProperty("pig.exec.reducers.max", "10");
String query = "a = load '/passwd';" +
- "b = order a by $0;" +
- "store b into 'output';";
+ "b = order a by $0;" +
+ "store b into 'output';";
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PhysicalPlan pp = Util.buildPp(ps, query);
@@ -714,17 +391,17 @@ public class TestJobSubmission {
//Third job is the order, which uses the estimated number of reducers
Util.assertParallelValues(-1, -1, reducer, reducer, jobControl.getWaitingJobs().get(0).getJobConf());
}
-
+
@Test
public void testToUri() throws Exception {
Class<JobControlCompiler> jobControlCompilerClass = JobControlCompiler.class;
Method toURIMethod = jobControlCompilerClass.getDeclaredMethod("toURI", Path.class);
toURIMethod.setAccessible(true);
-
+
Path p1 = new Path("/tmp/temp-1510081022/tmp-1308657145#pigsample_1889145873_1351808882314");
URI uri1 = (URI)toURIMethod.invoke(null, p1);
Assert.assertEquals(uri1.toString(), "/tmp/temp-1510081022/tmp-1308657145#pigsample_1889145873_1351808882314");
-
+
Path p2 = new Path("C:/Program Files/GnuWin32/bin/head.exe#pigsample_1889145873_1351808882314");
URI uri2 = (URI)toURIMethod.invoke(null, p2);
Assert.assertTrue(uri2.toString().equals("C:/Program%20Files/GnuWin32/bin/head.exe#pigsample_1889145873_1351808882314")||
Modified: pig/trunk/test/org/apache/pig/test/TestPackage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPackage.java?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPackage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPackage.java Sun Apr 6 10:54:13 2014
@@ -80,7 +80,7 @@ public class TestPackage {
// ITIterator iti = new TestPackage.ITIterator(db.iterator());
POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
pop.setNumInps(2);
- pop.setInner(inner);
+ pop.getPkgr().setInner(inner);
PigNullableWritable k = HDataType.getWritableComparableTypes(key, keyType);
pop.attachInput(k, db.iterator());
if (keyType != DataType.BAG) {
@@ -113,7 +113,7 @@ public class TestPackage {
new Pair<Boolean, Map<Integer, Integer>>(false, new HashMap<Integer, Integer>());
keyInfo.put(0, p);
keyInfo.put(1, p);
- pop.setKeyInfo(keyInfo);
+ pop.getPkgr().setKeyInfo(keyInfo);
Tuple t = null;
Result res = null;
res = pop.getNextTuple();
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld Sun Apr 6 10:54:13 2014
@@ -1,6 +1,6 @@
test-10: Store()
|
-|---A: Package[tuple]{tuple} - scope-19
+|---A: Package(Packager)[tuple]{tuple} - scope-19
|
|---A: Global Rearrange[tuple] - scope-18
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld Sun Apr 6 10:54:13 2014
@@ -5,7 +5,7 @@ MapReduce(-1) - -3:
| | |
| | Constant(true) - --3396897091865664764
| |
-| |---Package[tuple]{Unknown} - --5758282087831209061
+| |---Package(Packager)[tuple]{Unknown} - --5758282087831209061
| Local Rearrange[tuple]{Unknown}(false) - -3709512757404691843
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -2
@@ -13,7 +13,7 @@ MapReduce(-1) - -3:
|---MapReduce(-1) - -0:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| |
- | |---Package[tuple]{Unknown} - --2057425961601007773
+ | |---Package(Packager)[tuple]{Unknown} - --2057425961601007773
| Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
| |
| |---Load(DummyFil:DummyLdr) - -7506868571066332964
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld Sun Apr 6 10:54:13 2014
@@ -15,7 +15,7 @@ Reduce Plan Empty
| | | |
| | | Constant(true) - --8248982303554009
| | |
-| | |---Package[tuple]{Unknown} - -4061122832660258194
+| | |---Package(Packager)[tuple]{Unknown} - -4061122832660258194
| | Union[tuple] - -3
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - -3527883492192621891
@@ -33,7 +33,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -12:
| Store(file:/tmp/temp-1456742965/tmp774375955:org.apache.pig.impl.io.InterStorage) - -18
| |
- | |---Package[tuple]{Unknown} - -5679595123645092366
+ | |---Package(Packager)[tuple]{Unknown} - -5679595123645092366
| Union[tuple] - -13
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --8216215966586363937
@@ -47,7 +47,7 @@ Reduce Plan Empty
|---MapReduce(30) - -4:
| | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -5
| | |
- | | |---Package[tuple]{Unknown} - --7212359720440714287
+ | | |---Package(Packager)[tuple]{Unknown} - --7212359720440714287
| | Local Rearrange[tuple]{Unknown}(false) - -7469509242284658386
| | |
| | |---Load(DummyFil:DummyLdr) - -990040854696137546
@@ -55,7 +55,7 @@ Reduce Plan Empty
|---MapReduce(20) - -8:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -9
| |
- | |---Package[tuple]{Unknown} - --6259721534861268730
+ | |---Package(Packager)[tuple]{Unknown} - --6259721534861268730
| Local Rearrange[tuple]{Unknown}(false) - -3248199015665744565
| |
| |---Filter[tuple] - -6520791719738296531
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld Sun Apr 6 10:54:13 2014
@@ -15,7 +15,7 @@ Reduce Plan Empty
| | | |
| | | Constant(true) - -7391599663069134339
| | |
-| | |---Package[tuple]{Unknown} - -4024598850351165272
+| | |---Package(Packager)[tuple]{Unknown} - -4024598850351165272
| | Union[tuple] - -3
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - -913150185705910016
@@ -33,7 +33,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -14
| |
- | |---Package[tuple]{Unknown} - -3742910951635599848
+ | |---Package(Packager)[tuple]{Unknown} - -3742910951635599848
| Union[tuple] - -7
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -8
@@ -49,7 +49,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -4:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -9
| |
- | |---Package[tuple]{Unknown} - --5733160635931065595
+ | |---Package(Packager)[tuple]{Unknown} - --5733160635931065595
| Local Rearrange[tuple]{Unknown}(false) - --1115934782004129477
| |
| |---Load(DummyFil:DummyLdr) - -2833954415250116776
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld Sun Apr 6 10:54:13 2014
@@ -1,7 +1,7 @@
MapReduce(-1) - -6:
| Store(DummyFil:DummyLdr) - --1596982214453270401
| |
-| |---Package[tuple]{Unknown} - -2312201225366906989
+| |---Package(Packager)[tuple]{Unknown} - -2312201225366906989
| Union[tuple] - -7
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --4071762447953696591
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld Sun Apr 6 10:54:13 2014
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -8
| |
- | |---Package[tuple]{Unknown} - --1607475648664293401
+ | |---Package(Packager)[tuple]{Unknown} - --1607475648664293401
| Union[tuple] - -7
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --3476413267732334825
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld Sun Apr 6 10:54:13 2014
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(40) - -14:
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.impl.io.InterStorage) - -16
| |
- | |---Package[tuple]{Unknown} - --7096529877081178520
+ | |---Package(Packager)[tuple]{Unknown} - --7096529877081178520
| Union[tuple] - -15
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --7868505214447593853
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Sun Apr 6 10:54:13 2014
@@ -7,7 +7,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[tuple][*] - -3845271178065259412
| |
-| |---Package[tuple]{Unknown} - -3851605818031718348
+| |---Package(Packager)[tuple]{Unknown} - -3851605818031718348
| Local Rearrange[tuple]{Unknown}(false) - --3658165997714269604
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.impl.io.InterStorage) - -28
@@ -21,7 +21,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[tuple][*] - -2975419344702132532
| |
- | |---Package[tuple]{Unknown} - --368145047166239648
+ | |---Package(Packager)[tuple]{Unknown} - --368145047166239648
| Local Rearrange[tuple]{Unknown}(false) - -6541270116190953413
| |
| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -25
@@ -43,7 +43,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | Project[bag][1] - -22
| |
- | |---PackageLite[tuple]{tuple} - -21
+ | |---Package(LitePackager)[tuple]{tuple} - -21
| Local Rearrange[tuple]{tuple}(false) - -20
| | |
| | Project[tuple][*] - -19
@@ -69,7 +69,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[bag][1] - -9
| |
- | |---Package[tuple]{chararray} - -8
+ | |---Package(Packager)[tuple]{chararray} - -8
| Local Rearrange[tuple]{chararray}(false) - -7
| | |
| | Constant(all) - -6
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld Sun Apr 6 10:54:13 2014
@@ -5,7 +5,7 @@ MapReduce(-1) - -13:
| | |
| | Project[tuple][0] - -15
| |
-| |---Package[tuple]{tuple} - -14
+| |---Package(Packager)[tuple]{tuple} - -14
| Local Rearrange[tuple]{tuple}(true) - -10
| | |
| | Project[tuple][*] - -9
@@ -15,7 +15,7 @@ MapReduce(-1) - -13:
|---MapReduce(-1) - -8:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -11
| |
- | |---Package[tuple]{Unknown} - -3088212343542276753
+ | |---Package(Packager)[tuple]{Unknown} - -3088212343542276753
| Local Rearrange[tuple]{Unknown}(false) - --881122551222328650
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -7
@@ -27,7 +27,7 @@ MapReduce(-1) - -13:
| | |
| | Project[tuple][0] - -4
| |
- | |---Package[tuple]{tuple} - -3
+ | |---Package(Packager)[tuple]{tuple} - -3
| Local Rearrange[tuple]{tuple}(true) - -2
| | |
| | Project[tuple][*] - -1
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld Sun Apr 6 10:54:13 2014
@@ -7,11 +7,11 @@ MapReduce(1) - -0:
| | |
| | Project[tuple][1] - -4
| |
-| |---Package[tuple]{tuple} - -3
+| |---Package(Packager)[tuple]{tuple} - -3
| Local Rearrange[tuple]{tuple}(false) - -2
| | |
| | Project[tuple][*] - -1
| |
| |---Limit - --43001471365805096
| |
-| |---Load(DummyFil:DummyLdr) - -5534705358975373945
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - -5534705358975373945
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Sun Apr 6 10:54:13 2014
@@ -13,9 +13,9 @@ Reduce Plan Empty
| | |
| | Project[tuple][1] - scope-114
| |
- | |---Package[tuple]{tuple} - scope-113
+ | |---Package(Packager)[tuple]{tuple} - scope-113
| Local Rearrange[tuple]{tuple}(false) - scope-112
| | |
| | Project[tuple][*] - scope-111
| |
- | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','eNqtVTtvE0EQnlzixJiQhGeDKBCv7k6iQqKABIgwHNgiaXDF5G5zPti7XXb3wpkCiQYKKKFAAomCkt+AhCiooaRC9NSUMLu248MBUZgtLO/M7jy+/b65t9+hphXsv41b6Bcm5f6yUtgLU23Kh5+PvPiIr6ZhqgkzOr3PSgkAU/dm7C9dui5U4qPEqMt8mSb+BkZ3WB77XYyFkD4rWUQRRc7yJM3pSLen0wh5iD2mfMkx1357YGvTDvprygOvA3soUGtzmfNmLgsTQk3IDKWBYyElDfpJA0oapJnkgS08uFZwk15DebZUcGKsNHvKpfRbkik0QtmMT9/pl1/DD10P6iE0slUlsktxwvRdeADTlDO7ynrawGJo0RkkofghzGUhwy1GvqWKz4JGzpmsJV2IWgiz2Q0hjNvNhrCQrYlNM55m3lnXRdVWz6r7UhLaR//UknuxYeMDAD0PpmwVZHFVuNt7Rw98GXWXfLW5L+8/HLr1aRq8VWhwgfEqRgRME3aZrmK6K3hcynPnXcz5e3X6XbJ/S1dTY4fDMuL4P2EnRCvvfAW8NCagdSQkM7CvDyadT4I1o9I8OVsOu+qawTFKc3MS4hGLqTtN7mFNRMNWW4nbLDKj2mY7sJDqgZkeI4870BBbTFmkGG0OSiUiGyhPVjBpba4XkjPdgQMUnR5kjeycOSO5m1DXBpW5IDixJxK8yHJiz8EKe7Z1Z/m78b87vLTDNmoVKGE4ScKhineGnu9ADaOoyEjRqVXysjE2R9y0ON0tSEIsbqNCzhlPdbYCjT586z3JiBjIU3R8W6CNC2Dg8PgIiNFg4JAm9c26U0PF7eFULSZsXWHElIFT43
cly4firRykOHuFSqlValFEaDsfhpwThalkaEhULDd2nBhoj4cfwBj0YQzGYAx+gzFwIzGojkSqY1rZMXNxosA3HKSWV0f/KdDR27khQno1cPKPQ9deCyq6Jq1aTBYBftI6LWk5kNxyH41GKe0sWpmEa/1eKjNkBeYVM4XK1wyaQhMD+gz6fZy0NqyOqURvOFBgu7j+F62xPSXtBb+ZG5Ywte/b6zc/Hj4+49mPYG0LecEIxqXRuetFtsHUo7fPj+x+9vWJFZNLUJbj4e328F+Hp/M6+ModoDlD/S83YUI3yPIXqjl9HQ==','','b_45-1','scope','true')) - scope-102
\ No newline at end of file
+ | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','eNqtVTtvE0EQnlzixJiQhGeDKBCv7k6iQqKABIgwHNgiaXDF5G5zPti7XXb3wpkCiQYKKKFAAomCkt+AhCiooaRC9NSUMLu248MBUZgtLO/M7jy+/b65t9+hphXsv41b6Bcm5f6yUtgLU23Kh5+PvPiIr6ZhqgkzOr3PSgkAU/dm7C9dui5U4qPEqMt8mSb+BkZ3WB77XYyFkD4rWUQRRc7yJM3pSLen0wh5iD2mfMkx1357YGvTDvprygOvA3soUGtzmfNmLgsTQk3IDKWBYyElDfpJA0oapJnkgS08uFZwk15DebZUcGKsNHvKpfRbkik0QtmMT9/pl1/DD10P6iE0slUlsktxwvRdeADTlDO7ynrawGJo0RkkofghzGUhwy1GvqWKz4JGzpmsJV2IWgiz2Q0hjNvNhrCQrYlNM55m3lnXRdVWz6r7UhLaR//UknuxYeMDAD0PpmwVZHFVuNt7Rw98GXWXfLW5L+8/HLr1aRq8VWhwgfEqRgRME3aZrmK6K3hcynPnXcz5e3X6XbJ/S1dTY4fDMuL4P2EnRCvvfAW8NCagdSQkM7CvDyadT4I1o9I8OVsOu+qawTFKc3MS4hGLqTtN7mFNRMNWW4nbLDKj2mY7sJDqgZkeI4870BBbTFmkGG0OSiUiGyhPVjBpba4XkjPdgQMUnR5kjeycOSO5m1DXBpW5IDixJxK8yHJiz8EKe7Z1Z/m78b87vLTDNmoVKGE4ScKhineGnu9ADaOoyEjRqVXysjE2R9y0ON0tSEIsbqNCzhlPdbYCjT586z3JiBjIU3R8W6CNC2Dg8PgIiNFg4JAm9c26U0PF7eFULSZsXWHElIFT43
cly4firRykOHuFSqlValFEaDsfhpwThalkaEhULDd2nBhoj4cfwBj0YQzGYAx+gzFwIzGojkSqY1rZMXNxosA3HKSWV0f/KdDR27khQno1cPKPQ9deCyq6Jq1aTBYBftI6LWk5kNxyH41GKe0sWpmEa/1eKjNkBeYVM4XK1wyaQhMD+gz6fZy0NqyOqURvOFBgu7j+F62xPSXtBb+ZG5Ywte/b6zc/Hj4+49mPYG0LecEIxqXRuetFtsHUo7fPj+x+9vWJFZNLUJbj4e328F+Hp/M6+ModoDlD/S83YUI3yPIXqjl9HQ==','','b_45-1','scope','true')) - scope-102
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld Sun Apr 6 10:54:13 2014
@@ -5,7 +5,7 @@ MapReduce(1) - -0:
| | |
| | POSortedDistinct[tuple] - -1793365165218576787
| |
-| |---Package[tuple]{Unknown} - --3258087883799592471
+| |---Package(Packager)[tuple]{Unknown} - --3258087883799592471
| Local Rearrange[tuple]{Unknown}(false) - --8083579488061691196
| |
| |---Load(DummyFil:DummyLdr) - -698333249886773960
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld Sun Apr 6 10:54:13 2014
@@ -11,7 +11,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -4
| | |
-| | |---Package[tuple]{Unknown} - --2829086180578157275
+| | |---Package(Packager)[tuple]{Unknown} - --2829086180578157275
| | Local Rearrange[tuple]{Unknown}(false) - -6435233330554227045
| | |
| | |---Load(DummyFil:DummyLdr) - --4168060277593001906
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -1:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -6
| |
- | |---Package[tuple]{Unknown} - -7483213803049293823
+ | |---Package(Packager)[tuple]{Unknown} - -7483213803049293823
| Local Rearrange[tuple]{Unknown}(false) - --5190071275724378681
| |
| |---Load(DummyFil:DummyLdr) - --6957591191795645940
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld Sun Apr 6 10:54:13 2014
@@ -17,7 +17,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -5
| | |
-| | |---Package[tuple]{Unknown} - --3671186819751472084
+| | |---Package(Packager)[tuple]{Unknown} - --3671186819751472084
| | Local Rearrange[tuple]{Unknown}(false) - -3737603423295312892
| | |
| | |---Load(DummyFil:DummyLdr) - --5123390619301085966
@@ -25,7 +25,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -2:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -7
| |
- | |---Package[tuple]{Unknown} - --2325244147060806375
+ | |---Package(Packager)[tuple]{Unknown} - --2325244147060806375
| Local Rearrange[tuple]{Unknown}(false) - --1194577301115518934
| |
| |---Load(DummyFil:DummyLdr) - --8027742474430787324
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld Sun Apr 6 10:54:13 2014
@@ -23,7 +23,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -2:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -6
| | |
-| | |---Package[tuple]{Unknown} - -810744320494301308
+| | |---Package(Packager)[tuple]{Unknown} - -810744320494301308
| | Local Rearrange[tuple]{Unknown}(false) - --5111685507913827932
| | |
| | |---Load(DummyFil:DummyLdr) - --2426993543147308005
@@ -31,7 +31,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -3:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -8
| |
- | |---Package[tuple]{Unknown} - -7127277012934370361
+ | |---Package(Packager)[tuple]{Unknown} - -7127277012934370361
| Local Rearrange[tuple]{Unknown}(false) - --8622295867288126988
| |
| |---Load(DummyFil:DummyLdr) - -3683227376238667289
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld Sun Apr 6 10:54:13 2014
@@ -1,7 +1,7 @@
MapReduce(-1) - -8:
| Store(DummyFil:DummyLdr) - --6047015131487356012
| |
-| |---Package[tuple]{Unknown} - --2771799342014688397
+| |---Package(Packager)[tuple]{Unknown} - --2771799342014688397
| Union[tuple] - -9
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --1524666447399813617
@@ -15,7 +15,7 @@ MapReduce(-1) - -8:
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| | |
-| | |---Package[tuple]{Unknown} - -3777104251028634198
+| | |---Package(Packager)[tuple]{Unknown} - -3777104251028634198
| | Local Rearrange[tuple]{Unknown}(false) - --4160312061837144266
| | |
| | |---Load(DummyFil:DummyLdr) - -2997708366016271267
@@ -23,7 +23,7 @@ MapReduce(-1) - -8:
|---MapReduce(-1) - -4:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -5
| |
- | |---Package[tuple]{Unknown} - -9019572209815819418
+ | |---Package(Packager)[tuple]{Unknown} - -9019572209815819418
| Local Rearrange[tuple]{Unknown}(false) - -3615014757987062850
| |
| |---Load(DummyFil:DummyLdr) - --8706893458091286727
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld Sun Apr 6 10:54:13 2014
@@ -1,7 +1,7 @@
MapReduce(-1) - -9:
| Store(DummyFil:DummyLdr) - --4867358672373443663
| |
-| |---Package[tuple]{Unknown} - -31712229583931650
+| |---Package(Packager)[tuple]{Unknown} - -31712229583931650
| Union[tuple] - -10
| |
| |---Filter[tuple] - --4213306726552862637
@@ -21,7 +21,7 @@ MapReduce(-1) - -9:
|---MapReduce(-1) - -1:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -2
| | |
-| | |---Package[tuple]{Unknown} - --8479692259657755370
+| | |---Package(Packager)[tuple]{Unknown} - --8479692259657755370
| | Local Rearrange[tuple]{Unknown}(false) - -9193928674704944093
| | |
| | |---Load(DummyFil:DummyLdr) - --4238531569995320849
@@ -29,7 +29,7 @@ MapReduce(-1) - -9:
|---MapReduce(-1) - -5:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -6
| |
- | |---Package[tuple]{Unknown} - -8767305735755351861
+ | |---Package(Packager)[tuple]{Unknown} - -8767305735755351861
| Local Rearrange[tuple]{Unknown}(false) - -5965044993061572808
| |
| |---Load(DummyFil:DummyLdr) - -8914265632748254170
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld Sun Apr 6 10:54:13 2014
@@ -1,7 +1,7 @@
MapReduce(-1) - -10:
| Store(DummyFil:DummyLdr) - -7856319821130535798
| |
-| |---Package[tuple]{Unknown} - -7398260302074824818
+| |---Package(Packager)[tuple]{Unknown} - -7398260302074824818
| Union[tuple] - -11
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --7926255547935388282
@@ -27,7 +27,7 @@ MapReduce(-1) - -10:
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| | |
-| | |---Package[tuple]{Unknown} - --5683415113785058706
+| | |---Package(Packager)[tuple]{Unknown} - --5683415113785058706
| | Local Rearrange[tuple]{Unknown}(false) - -727770031531364881
| | |
| | |---Load(DummyFil:DummyLdr) - -7128285064986147947
@@ -35,7 +35,7 @@ MapReduce(-1) - -10:
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -7
| |
- | |---Package[tuple]{Unknown} - --885269774183211482
+ | |---Package(Packager)[tuple]{Unknown} - --885269774183211482
| Local Rearrange[tuple]{Unknown}(false) - --776319888013965510
| |
| |---Load(DummyFil:DummyLdr) - -7965768498188214494
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld?rev=1585283&r1=1585282&r2=1585283&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld Sun Apr 6 10:54:13 2014
@@ -1,7 +1,7 @@
MapReduce(-1) - -2:
| Store(DummyFil:DummyLdr) - -7714111612268358662
| |
-| |---Package[tuple]{Unknown} - --1613182091613226659
+| |---Package(Packager)[tuple]{Unknown} - --1613182091613226659
| Union[tuple] - -3
| |
| |---Filter[tuple] - -5165956429696944631