You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/11/28 09:56:34 UTC
svn commit: r1546314 [6/6] - in /pig/branches/tez:
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer...
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java Thu Nov 28 08:56:33 2013
@@ -6,37 +6,36 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
-import org.apache.pig.PigException;
import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
-import org.apache.pig.data.DataType;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ConstantExpression;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.CombinerPackager;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartialAgg;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPreCombinerLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.data.DataType;
import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.DepthFirstWalker;
-import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.PlanWalker;
import org.apache.pig.impl.plan.VisitorException;
-import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.plan.optimizer.OptimizerException;
import org.apache.pig.impl.util.Pair;
@@ -214,7 +213,10 @@ public class CombinerOptimizerUtil {
// Use the POCombiner package in the combine plan
// as it needs to act differently than the regular
// package operator.
- POCombinerPackage combinePack = new POCombinerPackage(pack, bags);
+ POPackage combinePack = pack.clone();
+ CombinerPackager combinePkgr = new CombinerPackager(
+ pack.getPkgr(), bags);
+ combinePack.setPkgr(combinePkgr);
combinePlan.add(combinePack);
combinePlan.add(cfe);
combinePlan.connect(combinePack, cfe);
@@ -251,19 +253,7 @@ public class CombinerOptimizerUtil {
// Change the package operator in the reduce plan to
// be the POCombiner package, as it needs to act
// differently than the regular package operator.
- POCombinerPackage newReducePack = new POCombinerPackage(pack, bags);
- reducePlan.replace(pack, newReducePack);
-
- // the replace() above only changes
- // the plan and does not change "inputs" to
- // operators
- // set up "inputs" for the operator after
- // package correctly
- List<PhysicalOperator> packList = Lists.newArrayList();
- packList.add(newReducePack);
- List<PhysicalOperator> sucs = reducePlan.getSuccessors(newReducePack);
- // there should be only one successor to package
- sucs.get(0).setInputs(packList);
+ pack.setPkgr(combinePkgr.clone());
} catch (Exception e) {
int errCode = 2018;
String msg = "Internal error. Unable to introduce the combiner for optimization.";
Modified: pig/branches/tez/src/org/apache/pig/data/ReadOnceBag.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/data/ReadOnceBag.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/data/ReadOnceBag.java (original)
+++ pig/branches/tez/src/org/apache/pig/data/ReadOnceBag.java Thu Nov 28 08:56:33 2013
@@ -22,33 +22,34 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Iterator;
+
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.LitePackager;
import org.apache.pig.impl.io.NullableTuple;
/**
- * This bag is specifically created for use by POPackageLite. So it has three
- * properties, the NullableTuple iterator, the key (Object) and the keyInfo
- * (Map<Integer, Pair<Boolean, Map<Integer, Integer>>>) all three
- * of which are required in the constructor call. This bag does not store
- * the tuples in memory, but has access to an iterator typically provided by
- * Hadoop. Use this when you already have an iterator over tuples and do not
+ * This bag is specifically created for use by POPackageLite. So it has three
+ * properties, the NullableTuple iterator, the key (Object) and the keyInfo
+ * (Map<Integer, Pair<Boolean, Map<Integer, Integer>>>) all three
+ * of which are required in the constructor call. This bag does not store
+ * the tuples in memory, but has access to an iterator typically provided by
+ * Hadoop. Use this when you already have an iterator over tuples and do not
* want to copy over again to a new bag.
*/
public class ReadOnceBag implements DataBag {
- // The Package operator that created this
- POPackageLite pkg;
-
+ // The Packager that created this
+ LitePackager pkgr;
+
//The iterator of Tuples. Marked transient because we will never serialize this.
transient Iterator<NullableTuple> tupIter;
-
+
// The key being worked on
Object key;
/**
- *
+ *
*/
private static final long serialVersionUID = 2L;
@@ -60,8 +61,9 @@ public class ReadOnceBag implements Data
* @param tupIter Iterator<NullableTuple>
* @param key Object
*/
- public ReadOnceBag(POPackageLite pkg, Iterator<NullableTuple> tupIter, Object key) {
- this.pkg = pkg;
+ public ReadOnceBag(LitePackager pkgr, Iterator<NullableTuple> tupIter,
+ Object key) {
+ this.pkgr = pkgr;
this.tupIter = tupIter;
this.key = key;
}
@@ -76,7 +78,7 @@ public class ReadOnceBag implements Data
/* (non-Javadoc)
* @see org.apache.pig.impl.util.Spillable#spill()
-
+
*/
@Override
public long spill() {
@@ -88,7 +90,7 @@ public class ReadOnceBag implements Data
*/
@Override
public void add(Tuple t) {
- throw new RuntimeException("ReadOnceBag does not support add operation");
+ throw new RuntimeException("ReadOnceBag does not support add operation");
}
/* (non-Javadoc)
@@ -132,7 +134,7 @@ public class ReadOnceBag implements Data
}
/* (non-Javadoc)
- * @see org.apache.pig.data.DataBag#markStale(boolean)
+ * @see org.apache.pig.data.DataBag#markStale(boolean)
*/
@Override
public void markStale(boolean stale) {
@@ -167,7 +169,7 @@ public class ReadOnceBag implements Data
/* (non-Javadoc)
* @see java.lang.Comparable#compareTo(java.lang.Object)
- * This has to be defined since DataBag implements
+ * This has to be defined since DataBag implements
* Comparable although, in this case we cannot really compare.
*/
@Override
@@ -177,27 +179,23 @@ public class ReadOnceBag implements Data
@Override
public boolean equals(Object other) {
- if(other instanceof ReadOnceBag)
- {
- if(pkg.getKeyTuple())
- {
- if(tupIter == ((ReadOnceBag)other).tupIter && pkg.getKeyTuple() == ((ReadOnceBag)other).pkg.getKeyTuple() && pkg.getKeyAsTuple().equals(((ReadOnceBag)other).pkg.getKeyAsTuple()))
- {
+ if (other instanceof ReadOnceBag) {
+ if (pkgr.getKeyTuple()) {
+ if (tupIter == ((ReadOnceBag) other).tupIter
+ && pkgr.getKeyTuple() == ((ReadOnceBag) other).pkgr
+ .getKeyTuple()
+ && pkgr.getKeyAsTuple().equals(
+ ((ReadOnceBag) other).pkgr.getKeyAsTuple())) {
return true;
- }
- else
- {
+ } else {
return false;
}
- }
- else
- {
- if(tupIter == ((ReadOnceBag)other).tupIter && pkg.getKey().equals(((ReadOnceBag)other).pkg.getKey()))
- {
+ } else {
+ if (tupIter == ((ReadOnceBag) other).tupIter
+ && pkgr.getKey().equals(
+ ((ReadOnceBag) other).pkgr.getKey())) {
return true;
- }
- else
- {
+ } else {
return false;
}
}
@@ -207,14 +205,14 @@ public class ReadOnceBag implements Data
@Override
public int hashCode() {
- int hash = 7;
- if(pkg.getKeyTuple())
+ int hash = 7;
+ if (pkgr.getKeyTuple())
{
- hash = hash*31 + pkg.getKeyAsTuple().hashCode();
+ hash = hash * 31 + pkgr.getKeyAsTuple().hashCode();
}
else
{
- hash = hash*31 + pkg.getKey().hashCode();
+ hash = hash * 31 + pkgr.getKey().hashCode();
}
return hash;
}
@@ -238,21 +236,21 @@ public class ReadOnceBag implements Data
int index = ntup.getIndex();
Tuple ret = null;
try {
- ret = pkg.getValueTuple(ntup, index, key);
+ ret = pkgr.getValueTuple(key, ntup, index);
} catch (ExecException e)
{
- throw new RuntimeException("ReadOnceBag failed to get value tuple : "+e.toString());
+ throw new RuntimeException("ReadOnceBag failed to get value tuple : "+e.toString());
}
return ret;
}
-
+
/* (non-Javadoc)
* @see java.util.Iterator#remove()
*/
@Override
public void remove() {
- throw new RuntimeException("ReadOnceBag.iterator().remove() is not allowed");
+ throw new RuntimeException("ReadOnceBag.iterator().remove() is not allowed");
}
- }
+ }
}
Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogToPhyTranslationVisitor.java Thu Nov 28 08:56:33 2013
@@ -56,7 +56,6 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PONative;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage.PackageType;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
@@ -64,6 +63,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.Packager.PackageType;
import org.apache.pig.data.DataType;
import org.apache.pig.data.SchemaTupleClassGenerator.GenContext;
import org.apache.pig.data.SchemaTupleFrontend;
@@ -218,7 +218,7 @@ public class LogToPhyTranslationVisitor
pushWalker(childWalker);
//currentWalker.walk(this);
currentWalker.walk(
- new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
+ new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
childWalker, filter, currentPlan, logToPhyMap ) );
popWalker();
@@ -260,7 +260,7 @@ public class LogToPhyTranslationVisitor
currentPlan = new PhysicalPlan();
PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(plan);
pushWalker(childWalker);
- childWalker.walk(new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
+ childWalker.walk(new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
childWalker, sort, currentPlan, logToPhyMap));
sortPlans.add(currentPlan);
popWalker();
@@ -357,7 +357,7 @@ public class LogToPhyTranslationVisitor
expressionPlans.put(i,loRank.getRankColPlans());
POPackage poPackage = compileToLR_GR_PackTrio(loRank, null, flags, expressionPlans);
- poPackage.setPackageType(PackageType.GROUP);
+ poPackage.getPkgr().setPackageType(PackageType.GROUP);
translateSoftLinks(loRank);
List<Boolean> flattenLst = Arrays.asList(true, false);
@@ -366,7 +366,7 @@ public class LogToPhyTranslationVisitor
POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)), -1);
feproj1.addOriginalLocation(loRank.getAlias(), loRank.getLocation());
feproj1.setColumn(0);
- feproj1.setResultType(poPackage.getKeyType());
+ feproj1.setResultType(poPackage.getPkgr().getKeyType());
feproj1.setStar(false);
feproj1.setOverloaded(false);
fep1.add(feproj1);
@@ -667,14 +667,14 @@ public class LogToPhyTranslationVisitor
throw new VisitorException(msg, errCode, PigException.BUG, e);
}
- poPackage.setKeyType(DataType.TUPLE);
+ poPackage.getPkgr().setKeyType(DataType.TUPLE);
poPackage.setResultType(DataType.TUPLE);
poPackage.setNumInps(count);
boolean inner[] = new boolean[count];
for (int i=0;i<count;i++) {
inner[i] = true;
}
- poPackage.setInner(inner);
+ poPackage.getPkgr().setInner(inner);
List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
List<Boolean> flattenLst = new ArrayList<Boolean>();
@@ -872,7 +872,7 @@ public class LogToPhyTranslationVisitor
poFE.addOriginalLocation(foreach.getAlias(), foreach.getLocation());
poFE.setResultType(DataType.BAG);
logToPhyMap.put(foreach, poFE);
- currentPlan.add(poFE);
+ currentPlan.add(poFE);
// generate cannot have multiple inputs
List<Operator> op = foreach.getPlan().getPredecessors(foreach);
@@ -977,8 +977,8 @@ public class LogToPhyTranslationVisitor
// } else {
// int errCode = 2051;
// String msg = "Did not find a predecessor for Store." ;
-// throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
- }
+// throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
+ }
try {
currentPlan.connect(from, store);
@@ -999,7 +999,7 @@ public class LogToPhyTranslationVisitor
break;
case REGULAR:
POPackage poPackage = compileToLR_GR_PackTrio(cg, cg.getCustomPartitioner(), cg.getInner(), cg.getExpressionPlans());
- poPackage.setPackageType(PackageType.GROUP);
+ poPackage.getPkgr().setPackageType(PackageType.GROUP);
logToPhyMap.put(cg, poPackage);
break;
case MERGE:
@@ -1350,8 +1350,8 @@ public class LogToPhyTranslationVisitor
SchemaTupleFrontend.registerToGenerateIfPossible(mergedSchema, false, GenContext.MERGE_JOIN);
}
- // inner join on two sorted inputs. We have less restrictive
- // implementation here in a form of POMergeJoin which doesn't
+ // inner join on two sorted inputs. We have less restrictive
+ // implementation here in a form of POMergeJoin which doesn't
// require loaders to implement collectable interface.
try {
smj = new POMergeJoin(new OperatorKey(scope,nodeGen.getNextNodeId(scope)),
@@ -1414,7 +1414,7 @@ public class LogToPhyTranslationVisitor
e.getErrorCode(),e.getErrorSource(),e);
}
logToPhyMap.put(loj, fe);
- poPackage.setPackageType(POPackage.PackageType.JOIN);
+ poPackage.getPkgr().setPackageType(PackageType.JOIN);
}
translateSoftLinks(loj);
}
@@ -1485,10 +1485,10 @@ public class LogToPhyTranslationVisitor
}
}
- poPackage.setKeyType(type);
+ poPackage.getPkgr().setKeyType(type);
poPackage.setResultType(DataType.TUPLE);
poPackage.setNumInps(count);
- poPackage.setInner(innerFlags);
+ poPackage.getPkgr().setInner(innerFlags);
return poPackage;
}
@@ -1502,7 +1502,7 @@ public class LogToPhyTranslationVisitor
try{
for(int i=0;i< inputs.size();i++){
PhysicalPlan fep1 = new PhysicalPlan();
- POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
+ POProject feproj1 = new POProject(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
parallel, i+1); //i+1 since the first column is the "group" field
feproj1.addOriginalLocation(alias, location);
feproj1.setResultType(DataType.BAG);
@@ -1656,8 +1656,8 @@ public class LogToPhyTranslationVisitor
} else {
int errCode = 2051;
String msg = "Did not find a predecessor for Split." ;
- throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
- }
+ throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG);
+ }
try {
currentPlan.connect(from, physOp);
@@ -1688,7 +1688,7 @@ public class LogToPhyTranslationVisitor
pushWalker(childWalker);
//currentWalker.walk(this);
currentWalker.walk(
- new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
+ new ExpToPhyTranslationVisitor( currentWalker.getPlan(),
childWalker, loSplitOutput, currentPlan, logToPhyMap) );
popWalker();
@@ -1722,7 +1722,7 @@ public class LogToPhyTranslationVisitor
* with as many null's as dictated by the schema
* @param fePlan the plan to update
* @param joinInput the relation for which the corresponding bag is being checked
- * @throws FrontendException
+ * @throws FrontendException
*/
public static void updateWithEmptyBagCheck(PhysicalPlan fePlan, Operator joinInput) throws FrontendException {
LogicalSchema inputSchema = null;
Modified: pig/branches/tez/src/org/apache/pig/pen/IllustratorAttacher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/pen/IllustratorAttacher.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/pen/IllustratorAttacher.java (original)
+++ pig/branches/tez/src/org/apache/pig/pen/IllustratorAttacher.java Thu Nov 28 08:56:33 2013
@@ -19,73 +19,63 @@
package org.apache.pig.pen;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.Map;
-import java.util.Collection;
import java.util.List;
+import java.util.Map;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCollectedGroup;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMultiQueryPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GreaterThanExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LessThanExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.EqualToExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GTOrEqualToExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.GreaterThanExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LTOrEqualToExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.EqualToExpr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.LessThanExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.NotEqualToExpr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORegexp;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POIsNull;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POAnd;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POOr;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONot;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POIsNull;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONegative;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONot;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POOr;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORegexp;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POJoinPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.LitePackager;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCounter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODemux;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PODistinct;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFRJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeCogroup;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSkewedJoin;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPartitionRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POOptimizedForEach;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPreCombinerLocalRearrange;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.PORank;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POUnion;
import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.PlanWalker;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.IdentityHashSet;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
-import org.apache.pig.pen.util.LineageTracer;
-import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.pen.util.LineageTracer;
/**
* The class used to (re)attach illustrators to physical operators
- *
+ *
*
*/
public class IllustratorAttacher extends PhyPlanVisitor {
@@ -206,20 +196,10 @@ public class IllustratorAttacher extends
@Override
public void visitPackage(POPackage pkg) throws VisitorException{
- if (!(pkg instanceof POPackageLite) && pkg.isDistinct())
- setIllustrator(pkg, 1);
- else
- setIllustrator(pkg, null);
- }
-
- @Override
- public void visitCombinerPackage(POCombinerPackage pkg) throws VisitorException{
- setIllustrator(pkg);
- }
-
- @Override
- public void visitMultiQueryPackage(POMultiQueryPackage pkg) throws VisitorException{
- setIllustrator(pkg);
+ if (!(pkg.getPkgr() instanceof LitePackager) && pkg.getPkgr().isDistinct())
+ setIllustrator(pkg, 1);
+ else
+ setIllustrator(pkg, null);
}
@Override
@@ -230,16 +210,17 @@ public class IllustratorAttacher extends
for (PhysicalPlan innerPlan : innerPlans)
innerPlanAttach(nfe, innerPlan);
List<PhysicalOperator> preds = mPlan.getPredecessors(nfe);
- if (preds != null && preds.size() == 1 &&
- preds.get(0) instanceof POPackage &&
- !(preds.get(0) instanceof POPackageLite) &&
- ((POPackage) preds.get(0)).isDistinct()) {
- // equivalence class of POPackage for DISTINCT needs to be used
- //instead of the succeeding POForEach's equivalence class
- setIllustrator(nfe, preds.get(0).getIllustrator().getEquivalenceClasses());
- nfe.getIllustrator().setEqClassesShared();
- } else
+ if (preds != null && preds.size() == 1 && preds.get(0) instanceof POPackage) {
+ POPackage pkg = (POPackage) preds.get(0);
+ if (!(pkg.getPkgr() instanceof LitePackager) && pkg.getPkgr().isDistinct()) {
+ // equivalence class of POPackage for DISTINCT needs to be used
+ // instead of the succeeding POForEach's equivalence class
+ setIllustrator(nfe, pkg.getIllustrator().getEquivalenceClasses());
+ nfe.getIllustrator().setEqClassesShared();
+ }
+ } else {
setIllustrator(nfe, 1);
+ }
}
@Override
@@ -269,14 +250,14 @@ public class IllustratorAttacher extends
}
@Override
- public void visitDistinct(PODistinct distinct) throws VisitorException {
+ public void visitDistinct(PODistinct distinct) throws VisitorException {
setIllustrator(distinct, 1);
- }
+ }
@Override
- public void visitSort(POSort sort) throws VisitorException {
+ public void visitSort(POSort sort) throws VisitorException {
setIllustrator(sort, 1);
- }
+ }
@Override
public void visitRank(PORank rank) throws VisitorException {
@@ -391,14 +372,6 @@ public class IllustratorAttacher extends
}
@Override
- public void visitJoinPackage(POJoinPackage joinPackage) throws VisitorException{
- if (revisit && joinPackage.getIllustrator() != null)
- return;
- setIllustrator(joinPackage);
- joinPackage.getForEach().setIllustrator(joinPackage.getIllustrator());
- }
-
- @Override
public void visitCast(POCast cast) {
}
Modified: pig/branches/tez/src/org/apache/pig/pen/PhysicalPlanResetter.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/pen/PhysicalPlanResetter.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/pen/PhysicalPlanResetter.java (original)
+++ pig/branches/tez/src/org/apache/pig/pen/PhysicalPlanResetter.java Thu Nov 28 08:56:33 2013
@@ -17,37 +17,24 @@
*/
package org.apache.pig.pen;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.PigException;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POJoinPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackageLite;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POCombinerPackage;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
-import org.apache.pig.impl.plan.optimizer.OptimizerException;
-import org.apache.pig.impl.util.Pair;
/**
* This visitor visits the physical plan and resets it for next MRCompilation
*/
public class PhysicalPlanResetter extends PhyPlanVisitor {
-
+
public PhysicalPlanResetter(PhysicalPlan plan) {
super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
}
-
+
@Override
public void visitPackage(POPackage pkg) throws VisitorException {
- pkg.setKeyInfo(null);
+ pkg.getPkgr().setKeyInfo(null);
}
}
Modified: pig/branches/tez/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestJobSubmission.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestJobSubmission.java Thu Nov 28 08:56:33 2013
@@ -100,7 +100,7 @@ public class TestJobSubmission {
if (Util.WINDOWS) {
inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
golDir="/"+FileLocalizer.parseCygPath(golDir, FileLocalizer.STYLE_WINDOWS);
- }
+ }
}
@After
@@ -112,329 +112,6 @@ public class TestJobSubmission {
cluster.shutDown();
}
-/* private void generateInput(int numTuples) throws ExecException{
-
- DataBag inpDb = GenRandomData.genRandSmallTupDataBag(r, numTuples, 1000);
-
- POProject proj = new POProject(new OperatorKey("", r.nextLong()));
- Tuple t = new DefaultTuple();
- t.append(inpDb);
- proj.attachInput(t);
- proj.setColumn(0);
- proj.setOverloaded(true);
- proj.setResultType(DataType.TUPLE);
-
- List<PhysicalOperator> inps = new ArrayList<PhysicalOperator>();
- inps.add(proj);
-
- POStore str = new POStore(new OperatorKey("", r.nextLong()));
- str.setInputs(inps);
-
- FileSpec fSpec = new FileSpec(ldFile, new FuncSpec(PigStorage.class.getName()));
-
- str.setSFile(fSpec);
- str.setPc(pc);
- str.store();
- }
-
- private void setUp1(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = ldFile;
- stFile = "jsTst1";
- grpName = "jobSubTst1";
-
- if(gen){
- generateInput(100);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- php.add(ld);
- php.add(st);
- php.connect(ld, st);
- }
-
-// @Test
- public void testCompile1() throws Exception {
- boolean gen = false;
-
- setUp1(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec fSpecAct = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp2(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst2.txt";
- expFile = ldFile;
- stFile = "jsTst2";
- grpName = "jobSubTst2";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- php.add(ld);
- php.add(st);
- php.connect(ld, st);
- }
-
-// @Test
- public void testCompile2() throws Exception {
- boolean gen = false;
-
- setUp2(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp3(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = "file:" + golDir + "jsTst3";
- stFile = "jsTst3";
- grpName = "jobSubTst3";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- int[] flds = {0,1};
- Tuple sample = new DefaultTuple();
- sample.append(new String("S"));
- sample.append(new Integer("10"));
-
- POForEach fe = GenPhyOp.topForEachOPWithPlan(flds , sample);
-
- POFilter fl = GenPhyOp.topFilterOpWithProj(1, 500, GenPhyOp.LT);
-
- php.add(ld);
- php.add(fe);
- php.connect(ld, fe);
-
- php.add(fl);
- php.connect(fe, fl);
-
- php.add(st);
- php.connect(fl, st);
- }
-
-// @Test
- public void testCompile3() throws Exception {
- boolean gen = false;
-
- setUp3(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
- }
-
- private void setUp4(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst1.txt";
- expFile = "file:" + golDir + "jsTst4";
- stFile = "jsTst4";
- grpName = "jobSubTst4";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- POSplit spl = GenPhyOp.topSplitOp();
- POFilter fl1 = GenPhyOp.topFilterOpWithProjWithCast(1, 200, GenPhyOp.LT);
- POFilter fl2 = GenPhyOp.topFilterOpWithProjWithCast(1, 800, GenPhyOp.GT);
-
- POUnion un = GenPhyOp.topUnionOp();
-
- php.add(ld);
- php.add(spl);
- php.connect(ld, spl);
-
- php.add(fl1);
- php.connect(spl, fl1);
-
- php.add(fl2);
- php.connect(spl, fl2);
-
- php.add(un);
- php.connect(fl1, un);
- php.connect(fl2, un);
-
- php.add(st);
- php.connect(un, st);
- }
-
-// @Test
- public void testCompile4() throws Exception {
- boolean gen = false;
-
- setUp4(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
-
- }
-
- private void setUp5(boolean gen) throws Exception {
- ldFile = "file:" + inpDir + "jsTst5.txt";
- expFile = ldFile;
- stFile = "jsTst5";
- grpName = "jobSubTst5";
-
- if(gen){
- generateInput(1000);
- return;
- }
-
- hadoopLdFile = FileLocalizer.hadoopify(ldFile, pc);
-
- FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
- POStore st = new POStore(new OperatorKey("", r.nextLong()));
- ld.setPc(pc);
- ld.setLFile(LFSpec);
- st.setPc(pc);
- st.setSFile(SFSpec);
-
- Tuple sample = new DefaultTuple();
- sample.append("S");
- sample.append(1);
- POLocalRearrange lr = GenPhyOp.topLocalRearrangeOPWithPlan(0, 1, sample);
-
- POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
-
- POPackage pk = GenPhyOp.topPackageOp();
- pk.setKeyType(DataType.INTEGER);
- pk.setNumInps(1);
- boolean[] inner = {false};
- pk.setInner(inner);
-
- POForEach fe = GenPhyOp.topForEachOPWithPlan(1);
-
- php.add(ld);
- php.add(lr);
- php.connect(ld, lr);
-
- php.add(gr);
- php.connect(lr, gr);
-
- php.add(pk);
- php.connect(gr, pk);
-
- php.add(fe);
- php.connect(pk, fe);
-
- php.add(st);
- php.connect(fe, st);
- }
-
- @Test
- public void testCompile5() throws Exception {
- boolean gen = false;
-
- setUp5(gen);
-
- if(gen)
- return;
-
- submit();
-
- assertEquals(true, FileLocalizer.fileExists(stFile, pc));
-
- FileSpec fSpecExp = new FileSpec(expFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
- FileSpec fSpecAct = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
-
- assertEquals(true, TestHelper.areFilesSame(fSpecExp, fSpecAct, pc));
-
- }*/
-
@Test
public void testJobControlCompilerErr() throws Exception {
String query = "a = load 'input';" + "b = order a by $0;" + "store b into 'output';";
@@ -443,17 +120,17 @@ public class TestJobSubmission {
POStore store = GenPhyOp.dummyPigStorageOp();
pp.addAsLeaf(store);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
-
+
for(MapReduceOper mro: mrPlan.getLeaves()) {
if(mro.reducePlan != null) {
PhysicalOperator po = mro.reducePlan.getRoots().get(0);
- if(po instanceof POPackage) {
- ((POPackage)po).setKeyType(DataType.BAG);
+ if (po instanceof POPackage) {
+ ((POPackage) po).getPkgr().setKeyType(DataType.BAG);
mro.setGlobalSort(true);
}
}
}
-
+
ConfigurationValidator.validatePigProperties(pc.getProperties());
Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
JobControlCompiler jcc = new JobControlCompiler(pc, conf);
@@ -714,17 +391,17 @@ public class TestJobSubmission {
//Third job is the order, which uses the estimated number of reducers
Util.assertParallelValues(-1, -1, reducer, reducer, jobControl.getWaitingJobs().get(0).getJobConf());
}
-
+
@Test
public void testToUri() throws Exception {
Class<JobControlCompiler> jobControlCompilerClass = JobControlCompiler.class;
Method toURIMethod = jobControlCompilerClass.getDeclaredMethod("toURI", Path.class);
toURIMethod.setAccessible(true);
-
+
Path p1 = new Path("/tmp/temp-1510081022/tmp-1308657145#pigsample_1889145873_1351808882314");
URI uri1 = (URI)toURIMethod.invoke(null, p1);
Assert.assertEquals(uri1.toString(), "/tmp/temp-1510081022/tmp-1308657145#pigsample_1889145873_1351808882314");
-
+
Path p2 = new Path("C:/Program Files/GnuWin32/bin/head.exe#pigsample_1889145873_1351808882314");
URI uri2 = (URI)toURIMethod.invoke(null, p2);
Assert.assertTrue(uri2.toString().equals("C:/Program%20Files/GnuWin32/bin/head.exe#pigsample_1889145873_1351808882314")||
Modified: pig/branches/tez/test/org/apache/pig/test/TestPackage.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPackage.java?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPackage.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPackage.java Thu Nov 28 08:56:33 2013
@@ -80,7 +80,7 @@ public class TestPackage {
// ITIterator iti = new TestPackage.ITIterator(db.iterator());
POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
pop.setNumInps(2);
- pop.setInner(inner);
+ pop.getPkgr().setInner(inner);
PigNullableWritable k = HDataType.getWritableComparableTypes(key, keyType);
pop.attachInput(k, db.iterator());
if (keyType != DataType.BAG) {
@@ -113,7 +113,7 @@ public class TestPackage {
new Pair<Boolean, Map<Integer, Integer>>(false, new HashMap<Integer, Integer>());
keyInfo.put(0, p);
keyInfo.put(1, p);
- pop.setKeyInfo(keyInfo);
+ pop.getPkgr().setKeyInfo(keyInfo);
Tuple t = null;
Result res = null;
res = pop.getNextTuple();
@@ -202,7 +202,6 @@ public class TestPackage {
continue;
}
System.out.println("Type " + DataType.findTypeName(b));
- boolean succ = true;
int NUM_TRIALS = 10;
boolean[] inner1 = { false, false };
for (int i = 0; i < NUM_TRIALS; i++)
@@ -219,4 +218,4 @@ public class TestPackage {
*/
}
}
-}
\ No newline at end of file
+}
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld Thu Nov 28 08:56:33 2013
@@ -1,6 +1,6 @@
test-10: Store()
|
-|---A: Package[tuple]{tuple} - scope-19
+|---A: Package(Packager)[tuple]{tuple} - scope-19
|
|---A: Global Rearrange[tuple] - scope-18
|
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld Thu Nov 28 08:56:33 2013
@@ -5,7 +5,7 @@ MapReduce(-1) - -3:
| | |
| | Constant(true) - --3396897091865664764
| |
-| |---Package[tuple]{Unknown} - --5758282087831209061
+| |---Package(Packager)[tuple]{Unknown} - --5758282087831209061
| Local Rearrange[tuple]{Unknown}(false) - -3709512757404691843
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -2
@@ -13,7 +13,7 @@ MapReduce(-1) - -3:
|---MapReduce(-1) - -0:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| |
- | |---Package[tuple]{Unknown} - --2057425961601007773
+ | |---Package(Packager)[tuple]{Unknown} - --2057425961601007773
| Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
| |
- | |---Load(DummyFil:DummyLdr) - -7506868571066332964
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -7506868571066332964
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld Thu Nov 28 08:56:33 2013
@@ -15,7 +15,7 @@ Reduce Plan Empty
| | | |
| | | Constant(true) - --8248982303554009
| | |
-| | |---Package[tuple]{Unknown} - -4061122832660258194
+| | |---Package(Packager)[tuple]{Unknown} - -4061122832660258194
| | Union[tuple] - -3
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - -3527883492192621891
@@ -33,7 +33,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -12:
| Store(file:/tmp/temp-1456742965/tmp774375955:org.apache.pig.impl.io.InterStorage) - -18
| |
- | |---Package[tuple]{Unknown} - -5679595123645092366
+ | |---Package(Packager)[tuple]{Unknown} - -5679595123645092366
| Union[tuple] - -13
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --8216215966586363937
@@ -47,7 +47,7 @@ Reduce Plan Empty
|---MapReduce(30) - -4:
| | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -5
| | |
- | | |---Package[tuple]{Unknown} - --7212359720440714287
+ | | |---Package(Packager)[tuple]{Unknown} - --7212359720440714287
| | Local Rearrange[tuple]{Unknown}(false) - -7469509242284658386
| | |
| | |---Load(DummyFil:DummyLdr) - -990040854696137546
@@ -55,11 +55,11 @@ Reduce Plan Empty
|---MapReduce(20) - -8:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -9
| |
- | |---Package[tuple]{Unknown} - --6259721534861268730
+ | |---Package(Packager)[tuple]{Unknown} - --6259721534861268730
| Local Rearrange[tuple]{Unknown}(false) - -3248199015665744565
| |
| |---Filter[tuple] - -6520791719738296531
| | |
| | Constant(true) - -2043312794799763441
| |
- | |---Load(DummyFil:DummyLdr) - --5314747545448923824
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --5314747545448923824
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld Thu Nov 28 08:56:33 2013
@@ -15,7 +15,7 @@ Reduce Plan Empty
| | | |
| | | Constant(true) - -7391599663069134339
| | |
-| | |---Package[tuple]{Unknown} - -4024598850351165272
+| | |---Package(Packager)[tuple]{Unknown} - -4024598850351165272
| | Union[tuple] - -3
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - -913150185705910016
@@ -33,7 +33,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -14
| |
- | |---Package[tuple]{Unknown} - -3742910951635599848
+ | |---Package(Packager)[tuple]{Unknown} - -3742910951635599848
| Union[tuple] - -7
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -8
@@ -49,7 +49,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -4:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -9
| |
- | |---Package[tuple]{Unknown} - --5733160635931065595
+ | |---Package(Packager)[tuple]{Unknown} - --5733160635931065595
| Local Rearrange[tuple]{Unknown}(false) - --1115934782004129477
| |
- | |---Load(DummyFil:DummyLdr) - -2833954415250116776
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -2833954415250116776
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld Thu Nov 28 08:56:33 2013
@@ -1,7 +1,7 @@
MapReduce(-1) - -6:
| Store(DummyFil:DummyLdr) - --1596982214453270401
| |
-| |---Package[tuple]{Unknown} - -2312201225366906989
+| |---Package(Packager)[tuple]{Unknown} - -2312201225366906989
| Union[tuple] - -7
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --4071762447953696591
@@ -24,4 +24,4 @@ MapReduce(-1) - -6:
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.impl.io.InterStorage) - -1
| |
- | |---Load(DummyFil:DummyLdr) - -2626971498845605352
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -2626971498845605352
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld Thu Nov 28 08:56:33 2013
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -8
| |
- | |---Package[tuple]{Unknown} - --1607475648664293401
+ | |---Package(Packager)[tuple]{Unknown} - --1607475648664293401
| Union[tuple] - -7
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --3476413267732334825
@@ -42,4 +42,4 @@ Reduce Plan Empty
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| |
- | |---Load(DummyFil:DummyLdr) - --7797378030368227515
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --7797378030368227515
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld Thu Nov 28 08:56:33 2013
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(40) - -14:
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.impl.io.InterStorage) - -16
| |
- | |---Package[tuple]{Unknown} - --7096529877081178520
+ | |---Package(Packager)[tuple]{Unknown} - --7096529877081178520
| Union[tuple] - -15
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --7868505214447593853
@@ -76,4 +76,4 @@ Reduce Plan Empty
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| |
- | |---Load(DummyFil:DummyLdr) - -8816201798088731301
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -8816201798088731301
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Thu Nov 28 08:56:33 2013
@@ -7,7 +7,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[tuple][*] - -5692939642430999118
| |
-| |---Package[tuple]{Unknown} - -6079615556647418436
+| |---Package(Packager)[tuple]{Unknown} - -6079615556647418436
| Local Rearrange[tuple]{Unknown}(false) - -8219725798912083822
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.impl.io.InterStorage) - -28
@@ -21,7 +21,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[tuple][*] - -7714111612268358662
| |
- | |---Package[tuple]{Unknown} - --1613182091613226659
+ | |---Package(Packager)[tuple]{Unknown} - --1613182091613226659
| Local Rearrange[tuple]{Unknown}(false) - -5165956429696944631
| |
| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.impl.io.InterStorage) - -25
@@ -43,7 +43,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | Project[bag][1] - -22
| |
- | |---PackageLite[tuple]{tuple} - -21
+ | |---Package(LitePackager)[tuple]{tuple} - -21
| Local Rearrange[tuple]{tuple}(false) - -20
| | |
| | Project[tuple][*] - -19
@@ -69,7 +69,7 @@ MapReduce(1,GFCross) - -29:
| | |
| | |---Project[bag][1] - -9
| |
- | |---Package[tuple]{chararray} - -8
+ | |---Package(Packager)[tuple]{chararray} - -8
| Local Rearrange[tuple]{chararray}(false) - -7
| | |
| | Constant(all) - -6
@@ -88,4 +88,4 @@ MapReduce(1,GFCross) - -29:
| | |
| | Constant(true) - --6860857982727545176
| |
- | |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -737965411848544085
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld Thu Nov 28 08:56:33 2013
@@ -5,7 +5,7 @@ MapReduce(-1) - -13:
| | |
| | Project[tuple][0] - -15
| |
-| |---Package[tuple]{tuple} - -14
+| |---Package(Packager)[tuple]{tuple} - -14
| Local Rearrange[tuple]{tuple}(true) - -10
| | |
| | Project[tuple][*] - -9
@@ -15,7 +15,7 @@ MapReduce(-1) - -13:
|---MapReduce(-1) - -8:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -11
| |
- | |---Package[tuple]{Unknown} - -2975419344702132532
+ | |---Package(Packager)[tuple]{Unknown} - -2975419344702132532
| Local Rearrange[tuple]{Unknown}(false) - -6555138338004402415
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -7
@@ -27,7 +27,7 @@ MapReduce(-1) - -13:
| | |
| | Project[tuple][0] - -4
| |
- | |---Package[tuple]{tuple} - -3
+ | |---Package(Packager)[tuple]{tuple} - -3
| Local Rearrange[tuple]{tuple}(true) - -2
| | |
| | Project[tuple][*] - -1
@@ -36,4 +36,4 @@ MapReduce(-1) - -13:
| | |
| | Constant(true) - -7251706569645724914
| |
- | |---Load(DummyFil:DummyLdr) - --7448840386612739853
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --7448840386612739853
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld Thu Nov 28 08:56:33 2013
@@ -7,11 +7,11 @@ MapReduce(1) - -0:
| | |
| | Project[tuple][1] - -4
| |
-| |---Package[tuple]{tuple} - -3
+| |---Package(Packager)[tuple]{tuple} - -3
| Local Rearrange[tuple]{tuple}(false) - -2
| | |
| | Project[tuple][*] - -1
| |
| |---Limit - --8049873144002881309
| |
-| |---Load(DummyFil:DummyLdr) - --3658165997714269604
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - --3658165997714269604
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Thu Nov 28 08:56:33 2013
@@ -13,9 +13,9 @@ Reduce Plan Empty
| | |
| | Project[tuple][1] - scope-114
| |
- | |---Package[tuple]{tuple} - scope-113
+ | |---Package(Packager)[tuple]{tuple} - scope-113
| Local Rearrange[tuple]{tuple}(false) - scope-112
| | |
| | Project[tuple][*] - scope-111
| |
- | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','eNqtVTtvE0EQnlzixJiQhGeDKBCv7k6iQqKABIgwHNgiaXDF5G5zPti7XXb3wpkCiQYKKKFAAomCkt+AhCiooaRC9NSUMLu248MBUZgtLO/M7jy+/b65t9+hphXsv41b6Bcm5f6yUtgLU23Kh5+PvPiIr6ZhqgkzOr3PSgkAU/dm7C9dui5U4qPEqMt8mSb+BkZ3WB77XYyFkD4rWUQRRc7yJM3pSLen0wh5iD2mfMkx1357YGvTDvprygOvA3soUGtzmfNmLgsTQk3IDKWBYyElDfpJA0oapJnkgS08uFZwk15DebZUcGKsNHvKpfRbkik0QtmMT9/pl1/DD10P6iE0slUlsktxwvRdeADTlDO7ynrawGJo0RkkofghzGUhwy1GvqWKz4JGzpmsJV2IWgiz2Q0hjNvNhrCQrYlNM55m3lnXRdVWz6r7UhLaR//UknuxYeMDAD0PpmwVZHFVuNt7Rw98GXWXfLW5L+8/HLr1aRq8VWhwgfEqRgRME3aZrmK6K3hcynPnXcz5e3X6XbJ/S1dTY4fDMuL4P2EnRCvvfAW8NCagdSQkM7CvDyadT4I1o9I8OVsOu+qawTFKc3MS4hGLqTtN7mFNRMNWW4nbLDKj2mY7sJDqgZkeI4870BBbTFmkGG0OSiUiGyhPVjBpba4XkjPdgQMUnR5kjeycOSO5m1DXBpW5IDixJxK8yHJiz8EKe7Z1Z/m78b87vLTDNmoVKGE4ScKhineGnu9ADaOoyEjRqVXysjE2R9y0ON0tSEIsbqNCzhlPdbYCjT586z3JiBjIU3R8W6CNC2Dg8PgIiNFg4JAm9c26U0PF7eFULSZsXWHElIFT43
cly4firRykOHuFSqlValFEaDsfhpwThalkaEhULDd2nBhoj4cfwBj0YQzGYAx+gzFwIzGojkSqY1rZMXNxosA3HKSWV0f/KdDR27khQno1cPKPQ9deCyq6Jq1aTBYBftI6LWk5kNxyH41GKe0sWpmEa/1eKjNkBeYVM4XK1wyaQhMD+gz6fZy0NqyOqURvOFBgu7j+F62xPSXtBb+ZG5Ywte/b6zc/Hj4+49mPYG0LecEIxqXRuetFtsHUo7fPj+x+9vWJFZNLUJbj4e328F+Hp/M6+ModoDlD/S83YUI3yPIXqjl9HQ==','','b_43-1','scope','true')) - scope-102
\ No newline at end of file
+ | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','eNqtVb1v00AUf3WbNoS2tHwtiAHxNWFLLCAxQFuoCJgmaruQiVf76ricfcf5XBwGpC4wsMLAgMTAyMg/gBiYYWRC7EhsjPDukjRuCmIIN0S59+7ex+9+v+e336GSKTiyhdvo5jrm7oJS2PHjTBc7n0++/IivxmGsDhNZ/IgVEgDGHk6YX7q0IlTkosSgzVwZR+4GBvdZGrptDIWQLitYQBFFytIoTulIu5PFAXIfO0y5kmOauc2erUk76K4xB5wWzFCgxuYC5/VU5tqHipAJSg2nfUrqdZN6lNSLE8k9U7h3J+c6voPySqHg7FBp5pRN6TYkU6iFMhl/4IWlS0/eLThQ9aGWLCuR3Agjlj2AxzBOOZPbrJNpOOQbdHpJKL4PU4nPcJuRb67kM6CRcyJpSBui4sNksiqEtrtJH2aTNbGph9NMW+u6KNuqSXlfSEL71J9asi/Wb7wHoOPAmKmCLLYKe3t+8MA3MWuTrzL15f2H4/c+jYOzDDUuMFzGgICpwwHdVixrCx4W8uo1G3P6YZV+58zfwtZU2+cwjDjzT9gJ0dI73wInDgnoLBCSaTjcBZPOR96aVnEaXSn6XbV17xiluTsK8YjF1F1G7n5NRMNGU4ktFuhBbZMtmI2znpkeIw1bUBPbTBmkGG2OSSUCEyiNFjFqbK7nkrOsBUcpOj3IGtk5s0Zy16GaaVR6SXBiTyB4nqTEnmMl9uzqzvB34393eGOfbdAqUEJ/lIR9Fe8PPd2CCgZBnpCiY6PkBa1NjrBucHqQk4RY2ESFnDMeZ8ki1LrwrXckI2Igj9HybZY2NoCGE8MjIESNnkWa1DdpT/UVN8OpWozYusKAKQ
3nh+9KlvbFWzpIceaFiqlValEEaDrvh5wSuS5lqElULNVmnGhoDofvweh1YfSGYPT2wOjZkeiVRyLVMa7MmLk+UuBVC6nh1al/CnTwdnaIkF41nPvj0DXXvJKuSasGk0MAv2hdlLQsSHbZj0atkGYWLY7CtW4vpRmyCNOK6Vylaxp1nhEDugzaO04aG0bHVKLTHyiwW1z3i1bbnZLmgltPNYuYOvzt9ZufO08vO+YjWNlGnjOCcW5wbiVPNph68vbFyYPPvz4zYrIJimI4vNme+OvwtF4LX7EPNGuo/uUmjOgGWfwG+uh74A==','','b_43-1','scope','true')) - scope-102
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC19.gld Thu Nov 28 08:56:33 2013
@@ -5,7 +5,7 @@ MapReduce(1) - -0:
| | |
| | POSortedDistinct[tuple] - --881122551222328650
| |
-| |---Package[tuple]{Unknown} - -49912403876500969
+| |---Package(Packager)[tuple]{Unknown} - -49912403876500969
| Local Rearrange[tuple]{Unknown}(false) - --790930717481674068
| |
-| |---Load(DummyFil:DummyLdr) - -4086340370247387675
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - -4086340370247387675
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld Thu Nov 28 08:56:33 2013
@@ -11,7 +11,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -4
| | |
-| | |---Package[tuple]{Unknown} - --2829086180578157275
+| | |---Package(Packager)[tuple]{Unknown} - --2829086180578157275
| | Local Rearrange[tuple]{Unknown}(false) - -6435233330554227045
| | |
| | |---Load(DummyFil:DummyLdr) - --4168060277593001906
@@ -19,7 +19,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -1:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -6
| |
- | |---Package[tuple]{Unknown} - -7483213803049293823
+ | |---Package(Packager)[tuple]{Unknown} - -7483213803049293823
| Local Rearrange[tuple]{Unknown}(false) - --5190071275724378681
| |
- | |---Load(DummyFil:DummyLdr) - --6957591191795645940
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --6957591191795645940
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld Thu Nov 28 08:56:33 2013
@@ -17,7 +17,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -5
| | |
-| | |---Package[tuple]{Unknown} - --3671186819751472084
+| | |---Package(Packager)[tuple]{Unknown} - --3671186819751472084
| | Local Rearrange[tuple]{Unknown}(false) - -3737603423295312892
| | |
| | |---Load(DummyFil:DummyLdr) - --5123390619301085966
@@ -25,7 +25,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -2:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -7
| |
- | |---Package[tuple]{Unknown} - --2325244147060806375
+ | |---Package(Packager)[tuple]{Unknown} - --2325244147060806375
| Local Rearrange[tuple]{Unknown}(false) - --1194577301115518934
| |
- | |---Load(DummyFil:DummyLdr) - --8027742474430787324
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --8027742474430787324
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC4.gld Thu Nov 28 08:56:33 2013
@@ -15,7 +15,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -87:
| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -91
| | |
-| | |---Package[tuple]{Unknown} - --6061281703859425960
+| | |---Package(Packager)[tuple]{Unknown} - --6061281703859425960
| | Local Rearrange[tuple]{Unknown}(false) - --1158897849427419546
| | |
| | |---Load(DummyFil:DummyLdr) - -3709512757404691843
@@ -23,7 +23,7 @@ Reduce Plan Empty
|---MapReduce(-1) - -88:
| Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -93
| |
- | |---Package[tuple]{Unknown} - --2057425961601007773
+ | |---Package(Packager)[tuple]{Unknown} - --2057425961601007773
| Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
| |
- | |---Load(DummyFil:DummyLdr) - -7506868571066332964
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -7506868571066332964
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld Thu Nov 28 08:56:33 2013
@@ -1,7 +1,7 @@
MapReduce(-1) - -8:
| Store(DummyFil:DummyLdr) - -8554221887998033529
| |
-| |---Package[tuple]{Unknown} - -5969405952749301492
+| |---Package(Packager)[tuple]{Unknown} - -5969405952749301492
| Union[tuple] - -9
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --203604194309414000
@@ -15,7 +15,7 @@ MapReduce(-1) - -8:
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| | |
-| | |---Package[tuple]{Unknown} - --2216790928607759809
+| | |---Package(Packager)[tuple]{Unknown} - --2216790928607759809
| | Local Rearrange[tuple]{Unknown}(false) - -7219092146411102494
| | |
| | |---Load(DummyFil:DummyLdr) - -4266652190633805129
@@ -23,7 +23,7 @@ MapReduce(-1) - -8:
|---MapReduce(-1) - -4:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -5
| |
- | |---Package[tuple]{Unknown} - -8957152010410190649
+ | |---Package(Packager)[tuple]{Unknown} - -8957152010410190649
| Local Rearrange[tuple]{Unknown}(false) - --2465563180022385815
| |
- | |---Load(DummyFil:DummyLdr) - --506205335735031954
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --506205335735031954
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld Thu Nov 28 08:56:33 2013
@@ -1,7 +1,7 @@
MapReduce(-1) - -9:
| Store(DummyFil:DummyLdr) - --6047015131487356012
| |
-| |---Package[tuple]{Unknown} - --2771799342014688397
+| |---Package(Packager)[tuple]{Unknown} - --2771799342014688397
| Union[tuple] - -10
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --4160312061837144266
@@ -21,7 +21,7 @@ MapReduce(-1) - -9:
|---MapReduce(-1) - -0:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -1
| | |
-| | |---Package[tuple]{Unknown} - --8706893458091286727
+| | |---Package(Packager)[tuple]{Unknown} - --8706893458091286727
| | Local Rearrange[tuple]{Unknown}(false) - -6693653468874366867
| | |
| | |---Load(DummyFil:DummyLdr) - --3347320220748403847
@@ -29,7 +29,7 @@ MapReduce(-1) - -9:
|---MapReduce(-1) - -5:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -6
| |
- | |---Package[tuple]{Unknown} - -2997708366016271267
+ | |---Package(Packager)[tuple]{Unknown} - -2997708366016271267
| Local Rearrange[tuple]{Unknown}(false) - --1195368474144577564
| |
- | |---Load(DummyFil:DummyLdr) - -3615014757987062850
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -3615014757987062850
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld Thu Nov 28 08:56:33 2013
@@ -1,7 +1,7 @@
MapReduce(-1) - -10:
| Store(DummyFil:DummyLdr) - --2711892325794580872
| |
-| |---Package[tuple]{Unknown} - --776319888013965510
+| |---Package(Packager)[tuple]{Unknown} - --776319888013965510
| Union[tuple] - -11
| |
| |---Filter[tuple] - --4867358672373443663
@@ -27,7 +27,7 @@ MapReduce(-1) - -10:
|---MapReduce(-1) - -2:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.io.InterStorage) - -3
| | |
-| | |---Package[tuple]{Unknown} - --8479692259657755370
+| | |---Package(Packager)[tuple]{Unknown} - --8479692259657755370
| | Local Rearrange[tuple]{Unknown}(false) - -9193928674704944093
| | |
| | |---Load(DummyFil:DummyLdr) - --4238531569995320849
@@ -35,7 +35,7 @@ MapReduce(-1) - -10:
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.impl.io.InterStorage) - -7
| |
- | |---Package[tuple]{Unknown} - -8767305735755351861
+ | |---Package(Packager)[tuple]{Unknown} - -8767305735755351861
| Local Rearrange[tuple]{Unknown}(false) - -5965044993061572808
| |
- | |---Load(DummyFil:DummyLdr) - -8914265632748254170
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -8914265632748254170
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld Thu Nov 28 08:56:33 2013
@@ -1,7 +1,7 @@
MapReduce(-1) - -2:
| Store(DummyFil:DummyLdr) - --3148893660811981376
| |
-| |---Package[tuple]{Unknown} - --3359714221559579809
+| |---Package(Packager)[tuple]{Unknown} - --3359714221559579809
| Union[tuple] - -3
| |
| |---Filter[tuple] - -727770031531364881
@@ -14,4 +14,4 @@ MapReduce(-1) - -2:
| | |
| | Constant(true) - --7926255547935388282
| |
-| |---Load(DummyFil:DummyLdr) - --5683415113785058706
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - --5683415113785058706
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld Thu Nov 28 08:56:33 2013
@@ -31,4 +31,4 @@ c: Store(file:///tmp/output:org.apache.p
| |
| Project[bag][1] - scope-32
|
- |---b: Package[tuple]{int} - scope-27
+ |---b: Package(Packager)[tuple]{int} - scope-27
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld Thu Nov 28 08:56:33 2013
@@ -56,4 +56,4 @@ d: Store(file:///tmp/output:org.apache.p
| |
| Project[bag][2] - scope-64
|
- |---c: Package[tuple]{int} - scope-58
+ |---c: Package(Packager)[tuple]{int} - scope-58
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld Thu Nov 28 08:56:33 2013
@@ -33,4 +33,4 @@ c: Store(file:///tmp/output:org.apache.p
| |
| Project[tuple][1] - scope-93
|
- |---Package[tuple]{tuple} - scope-92
+ |---Package(Packager)[tuple]{tuple} - scope-92
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld?rev=1546314&r1=1546313&r2=1546314&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld Thu Nov 28 08:56:33 2013
@@ -33,4 +33,4 @@ c: Store(file:///tmp/output:org.apache.p
| |
| Project[tuple][0] - scope-115
|
- |---Package[tuple]{tuple} - scope-114
+ |---Package(Packager)[tuple]{tuple} - scope-114