You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/12/17 23:08:09 UTC
svn commit: r1050503 [2/2] - in /pig/trunk: src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/ ...
Modified: pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/ExampleGenerator.java Fri Dec 17 22:08:08 2010
@@ -41,14 +41,14 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.PigException;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.LOSort;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.Operator;
import org.apache.pig.pen.util.DisplayExamples;
import org.apache.pig.pen.util.LineageTracer;
@@ -60,6 +60,7 @@ import org.apache.pig.pen.util.LineageTr
public class ExampleGenerator {
LogicalPlan plan;
+ org.apache.pig.newplan.logical.relational.LogicalPlan newPlan;
Map<LOLoad, DataBag> baseData = null;
PigContext pigContext;
@@ -72,16 +73,16 @@ public class ExampleGenerator {
private int MAX_RECORDS = 10000;
- private Map<LogicalOperator, PhysicalOperator> logToPhyMap;
- private Map<PhysicalOperator, LogicalOperator> poLoadToLogMap;
- private Map<PhysicalOperator, LogicalOperator> poToLogMap;
+ private Map<Operator, PhysicalOperator> logToPhyMap;
+ private Map<PhysicalOperator, Operator> poLoadToLogMap;
+ private Map<PhysicalOperator, Operator> poToLogMap;
private HashMap<PhysicalOperator, Collection<IdentityHashSet<Tuple>>> poToEqclassesMap;
private LineageTracer lineage;
- private Map<LogicalOperator, DataBag> logToDataMap = null;
- private Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap;
- Map<LOForEach, Map<LogicalOperator, PhysicalOperator>> forEachInnerLogToPhyMap;
+ private Map<Operator, DataBag> logToDataMap = null;
+ private Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap;
+ Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> forEachInnerLogToPhyMap;
Map<LOLimit, Long> oriLimitMap = null;
- Map<POLoad, Schema> poLoadToSchemaMap;
+ Map<POLoad, LogicalSchema> poLoadToSchemaMap;
public ExampleGenerator(LogicalPlan plan, PigContext hadoopPigContext) {
this.plan = plan;
@@ -99,14 +100,14 @@ public class ExampleGenerator {
}
execEngine = new HExecutionEngine(pigContext);
localMRRunner = new LocalMapReduceSimulator();
- poLoadToSchemaMap = new HashMap<POLoad, Schema>();
+ poLoadToSchemaMap = new HashMap<POLoad, LogicalSchema>();
}
public LineageTracer getLineage() {
return lineage;
}
- public Map<LogicalOperator, PhysicalOperator> getLogToPhyMap() {
+ public Map<Operator, PhysicalOperator> getLogToPhyMap() {
return logToPhyMap;
}
@@ -114,36 +115,36 @@ public class ExampleGenerator {
MAX_RECORDS = max;
}
- public Map<LogicalOperator, DataBag> getExamples() throws IOException, InterruptedException {
+ public Map<Operator, DataBag> getExamples() throws IOException, InterruptedException {
if (pigContext.getProperties().getProperty("pig.usenewlogicalplan", "true").equals("false"))
throw new ExecException("ILLUSTRATE must use the new logical plan!");
pigContext.inIllustrator = true;
physPlan = compilePlan(plan);
physPlanReseter = new PhysicalPlanResetter(physPlan);
- List<LogicalOperator> loads = plan.getRoots();
+ List<Operator> loads = newPlan.getSources();
List<PhysicalOperator> pRoots = physPlan.getRoots();
if (loads.size() != pRoots.size())
throw new ExecException("Logical and Physical plans have different number of roots");
logToPhyMap = execEngine.getLogToPhyMap();
forEachInnerLogToPhyMap = execEngine.getForEachInnerLogToPhyMap();
- poLoadToLogMap = new HashMap<PhysicalOperator, LogicalOperator>();
- logToDataMap = new HashMap<LogicalOperator, DataBag>();
- poToLogMap = new HashMap<PhysicalOperator, LogicalOperator>();
+ poLoadToLogMap = new HashMap<PhysicalOperator, Operator>();
+ logToDataMap = new HashMap<Operator, DataBag>();
+ poToLogMap = new HashMap<PhysicalOperator, Operator>();
// set up foreach inner data map
- forEachInnerLogToDataMap = new HashMap<LOForEach, Map<LogicalOperator, DataBag>>();
- for (Map.Entry<LOForEach, Map<LogicalOperator, PhysicalOperator>> entry : forEachInnerLogToPhyMap.entrySet()) {
- Map<LogicalOperator, DataBag> innerMap = new HashMap<LogicalOperator, DataBag>();
+ forEachInnerLogToDataMap = new HashMap<LOForEach, Map<LogicalRelationalOperator, DataBag>>();
+ for (Map.Entry<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> entry : forEachInnerLogToPhyMap.entrySet()) {
+ Map<LogicalRelationalOperator, DataBag> innerMap = new HashMap<LogicalRelationalOperator, DataBag>();
forEachInnerLogToDataMap.put(entry.getKey(), innerMap);
}
- for (LogicalOperator load : loads)
+ for (Operator load : loads)
{
poLoadToLogMap.put(logToPhyMap.get(load), load);
}
boolean hasLimit = false;
- for (LogicalOperator lo : logToPhyMap.keySet()) {
+ for (Operator lo : logToPhyMap.keySet()) {
poToLogMap.put(logToPhyMap.get(lo), lo);
if (!hasLimit && lo instanceof LOLimit)
hasLimit = true;
@@ -159,64 +160,58 @@ public class ExampleGenerator {
throw new RuntimeException(e.getMessage());
}
- Map<LogicalOperator, DataBag> derivedData = null;
- try {
+ Map<Operator, DataBag> derivedData = null;
- // create derived data and trim base data
- LineageTrimmingVisitor trimmer = new LineageTrimmingVisitor(plan,
- baseData, this, logToPhyMap, physPlan, pigContext);
- trimmer.visit();
- // System.out.println(
- // "Obtained the first level derived and trimmed data");
- // create new derived data from trimmed basedata
- derivedData = getData(physPlan);
-
- // System.out.println(
- // "Got new derived data from the trimmed base data");
- // augment base data
- AugmentBaseDataVisitor augment = new AugmentBaseDataVisitor(plan,
- logToPhyMap, baseData, derivedData);
+ // create derived data and trim base data
+ LineageTrimmingVisitor trimmer = new LineageTrimmingVisitor(newPlan,
+ baseData, this, logToPhyMap, physPlan, pigContext);
+ trimmer.visit();
+ baseData = trimmer.getBaseData();
+ // System.out.println(
+ // "Obtained the first level derived and trimmed data");
+ // create new derived data from trimmed basedata
+ derivedData = getData(physPlan);
+
+ // System.out.println(
+ // "Got new derived data from the trimmed base data");
+ // augment base data
+ AugmentBaseDataVisitor augment = new AugmentBaseDataVisitor(newPlan,
+ logToPhyMap, baseData, derivedData);
+ augment.visit();
+ this.baseData = augment.getNewBaseData();
+ // System.out.println("Obtained augmented base data");
+ // create new derived data and trim the base data after augmenting
+ // base data with synthetic tuples
+ trimmer = new LineageTrimmingVisitor(newPlan, baseData, this,
+ logToPhyMap, physPlan, pigContext);
+ trimmer.visit();
+ baseData = trimmer.getBaseData();
+ // System.out.println("Final trimming");
+ // create the final version of derivedData to give to the output
+ derivedData = getData(physPlan);
+ // System.out.println("Obtaining final derived data for output");
+
+ if (hasLimit)
+ {
+ augment.setLimit();
augment.visit();
this.baseData = augment.getNewBaseData();
- // System.out.println("Obtained augmented base data");
- // create new derived data and trim the base data after augmenting
- // base data with synthetic tuples
- trimmer = new LineageTrimmingVisitor(plan, baseData, this,
- logToPhyMap, physPlan, pigContext);
- trimmer.visit();
- // System.out.println("Final trimming");
- // create the final version of derivedData to give to the output
- derivedData = getData(physPlan);
- // System.out.println("Obtaining final derived data for output");
-
- if (hasLimit)
- {
- augment.setLimit();
- augment.visit();
- this.baseData = augment.getNewBaseData();
- oriLimitMap = augment.getOriLimitMap();
- derivedData = getData();
- }
-
- } catch (VisitorException e) {
- e.printStackTrace(System.out);
- log.error("Visitor exception while creating example data "
- + e.getMessage());
- throw new RuntimeException(e.getMessage());
+ oriLimitMap = augment.getOriLimitMap();
+ derivedData = getData();
}
// DisplayExamples.printSimple(plan.getLeaves().get(0),
// derivedData.derivedData);
- System.out.println(DisplayExamples.printTabular(plan,
+ System.out.println(DisplayExamples.printTabular(newPlan,
derivedData, forEachInnerLogToDataMap));
pigContext.inIllustrator = false;
return derivedData;
}
- private void readBaseData(List<LogicalOperator> loads) throws IOException, InterruptedException, FrontendException, ExecException {
+ private void readBaseData(List<Operator> loads) throws IOException, InterruptedException, FrontendException, ExecException {
PhysicalPlan thisPhyPlan = new PhysicalPlan();
- for (LogicalOperator op : loads) {
- Schema schema = op.getSchema();
+ for (Operator op : loads) {
+ LogicalSchema schema = ((LOLoad) op).getSchema();
if(schema == null) {
throw new ExecException("Example Generator requires a schema. Please provide a schema while loading data.");
}
@@ -224,9 +219,9 @@ public class ExampleGenerator {
thisPhyPlan.add(logToPhyMap.get(op));
}
baseData = null;
- Map<LogicalOperator, DataBag> result = getData(thisPhyPlan);
+ Map<Operator, DataBag> result = getData(thisPhyPlan);
baseData = new HashMap<LOLoad, DataBag>();
- for (LogicalOperator lo : result.keySet()) {
+ for (Operator lo : result.keySet()) {
if (lo instanceof LOLoad) {
baseData.put((LOLoad) lo, result.get(lo));
}
@@ -234,14 +229,16 @@ public class ExampleGenerator {
}
PhysicalPlan compilePlan(LogicalPlan plan) throws ExecException, FrontendException {
- return execEngine.compile(plan, null);
+ PhysicalPlan result = execEngine.compile(plan, null);
+ newPlan = execEngine.getNewPlan();
+ return result;
}
- public Map<LogicalOperator, DataBag> getData() throws IOException, InterruptedException {
+ public Map<Operator, DataBag> getData() throws IOException, InterruptedException {
return getData(physPlan);
}
- private Map<LogicalOperator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
+ private Map<Operator, DataBag> getData(PhysicalPlan plan) throws PigException, IOException, InterruptedException
{
// get data on a physical plan possibly trimmed of one branch
lineage = new LineageTracer();
@@ -257,7 +254,7 @@ public class ExampleGenerator {
setLoadDataMap();
physPlanReseter.visit();
}
- localMRRunner.launchPig(plan, baseData, poLoadToLogMap, lineage, attacher, this, pigContext);
+ localMRRunner.launchPig(plan, baseData, lineage, attacher, this, pigContext);
if (baseData == null)
poToEqclassesMap = attacher.poToEqclassesMap;
else {
@@ -272,7 +269,7 @@ public class ExampleGenerator {
return logToDataMap;
}
- public Map<LogicalOperator, DataBag> getData(Map<LOLoad, DataBag> newBaseData) throws Exception
+ public Map<Operator, DataBag> getData(Map<LOLoad, DataBag> newBaseData) throws Exception
{
baseData = newBaseData;
return getData(physPlan);
@@ -281,7 +278,7 @@ public class ExampleGenerator {
private void phyToMRTransform(PhysicalPlan plan, Map<PhysicalOperator, DataBag> phyToDataMap) {
// remap the LO to PO as result of the MR compilation may have changed PO in the MR plans
Map<PhysicalOperator, PhysicalOperator> phyToMRMap = localMRRunner.getPhyToMRMap();
- for (Map.Entry<PhysicalOperator, LogicalOperator> entry : poToLogMap.entrySet()) {
+ for (Map.Entry<PhysicalOperator, Operator> entry : poToLogMap.entrySet()) {
if (phyToMRMap.get(entry.getKey()) != null) {
PhysicalOperator poInMR = phyToMRMap.get(entry.getKey());
logToDataMap.put(entry.getValue(), phyToDataMap.get(poInMR));
@@ -292,15 +289,15 @@ public class ExampleGenerator {
private void getLogToDataMap(Map<PhysicalOperator, DataBag> phyToDataMap) {
logToDataMap.clear();
- for (LogicalOperator lo : logToPhyMap.keySet()) {
+ for (Operator lo : logToPhyMap.keySet()) {
if (logToPhyMap.get(lo) != null)
logToDataMap.put(lo, phyToDataMap.get(logToPhyMap.get(lo)));
}
// set the LO-to-Data mapping for the ForEach inner plans
- for (Map.Entry<LOForEach, Map<LogicalOperator, DataBag>> entry : forEachInnerLogToDataMap.entrySet()) {
+ for (Map.Entry<LOForEach, Map<LogicalRelationalOperator, DataBag>> entry : forEachInnerLogToDataMap.entrySet()) {
entry.getValue().clear();
- for (Map.Entry<LogicalOperator, PhysicalOperator> innerEntry : forEachInnerLogToPhyMap.get(entry.getKey()).entrySet()) {
+ for (Map.Entry<LogicalRelationalOperator, PhysicalOperator> innerEntry : forEachInnerLogToPhyMap.get(entry.getKey()).entrySet()) {
entry.getValue().put(innerEntry.getKey(), phyToDataMap.get(innerEntry.getValue()));
}
}
@@ -328,10 +325,10 @@ public class ExampleGenerator {
}
}
- public Collection<IdentityHashSet<Tuple>> getEqClasses() throws VisitorException {
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> logToEqclassesMap = getLoToEqClassMap();
+ public Collection<IdentityHashSet<Tuple>> getEqClasses() {
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> logToEqclassesMap = getLoToEqClassMap();
LinkedList<IdentityHashSet<Tuple>> ret = new LinkedList<IdentityHashSet<Tuple>>();
- for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> entry :
+ for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> entry :
logToEqclassesMap.entrySet()) {
if (entry.getValue() != null)
ret.addAll(entry.getValue());
@@ -339,12 +336,12 @@ public class ExampleGenerator {
return ret;
}
- public Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> getLoToEqClassMap() throws VisitorException {
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> ret =
- EquivalenceClasses.getLoToEqClassMap(physPlan, plan, logToPhyMap, logToDataMap, forEachInnerLogToPhyMap, poToEqclassesMap);
+ public Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> getLoToEqClassMap() {
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> ret =
+ EquivalenceClasses.getLoToEqClassMap(physPlan, newPlan, logToPhyMap, logToDataMap, forEachInnerLogToPhyMap, poToEqclassesMap);
// eq classes adjustments based upon logical operators
- for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> entry :ret.entrySet())
+ for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> entry :ret.entrySet())
{
if (entry.getKey() instanceof LOSort) {
Collection<IdentityHashSet<Tuple>> eqClasses = entry.getValue();
Modified: pig/trunk/src/org/apache/pig/pen/Illustrator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/Illustrator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/Illustrator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/Illustrator.java Fri Dec 17 22:08:08 2010
@@ -29,7 +29,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.pen.util.LineageTracer;
import org.apache.pig.impl.util.IdentityHashSet;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
/**
* Class used by physical operators to generate example tuples for the ILLUSTRATE
@@ -51,7 +51,7 @@ public class Illustrator {
private Boolean[] subExpResult;
private boolean eqClassesShared;
private long oriLimit = -1;
- private Schema schema;
+ private LogicalSchema schema;
public Illustrator(LineageTracer lineage, LinkedList<IdentityHashSet<Tuple>> equivalenceClasses, IllustratorAttacher attacher, PigContext hadoopPigContext) {
this.lineage = lineage;
@@ -64,7 +64,7 @@ public class Illustrator {
}
public Illustrator(LineageTracer lineage, LinkedList<IdentityHashSet<Tuple>> equivalenceClasses, int maxRecords, IllustratorAttacher attacher,
- Schema schema, PigContext hadoopPigContext) {
+ LogicalSchema schema, PigContext hadoopPigContext) {
this(lineage, equivalenceClasses, attacher, hadoopPigContext);
this.maxRecords = maxRecords;
this.schema = schema;
@@ -136,7 +136,7 @@ public class Illustrator {
return eqClassesShared;
}
- public Schema getSchema() {
+ public LogicalSchema getSchema() {
return schema;
}
}
Modified: pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java (original)
+++ pig/trunk/src/org/apache/pig/pen/IllustratorAttacher.java Fri Dec 17 22:08:08 2010
@@ -79,7 +79,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.pen.util.LineageTracer;
import org.apache.pig.impl.plan.DepthFirstWalker;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
/**
* The class used to (re)attach illustrators to physical operators
@@ -98,10 +98,10 @@ public class IllustratorAttacher extends
private int maxRecords;
private boolean revisit = false;
private ArrayList<Boolean[]> subExpResults = null;
- private final Map<POLoad, Schema> poloadToSchemaMap;
+ private final Map<POLoad, LogicalSchema> poloadToSchemaMap;
public IllustratorAttacher(PhysicalPlan plan, LineageTracer lineage, int maxRecords,
- Map<POLoad, Schema> poLoadToSchemaMap, PigContext hadoopPigContext) throws VisitorException {
+ Map<POLoad, LogicalSchema> poLoadToSchemaMap, PigContext hadoopPigContext) throws VisitorException {
super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
pigContext = hadoopPigContext;
this.lineage = lineage;
Modified: pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/pen/LineageTrimmingVisitor.java Fri Dec 17 22:08:08 2010
@@ -39,22 +39,23 @@ import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.LOCogroup;
-import org.apache.pig.impl.logicalLayer.LOJoin;
-import org.apache.pig.impl.logicalLayer.LOCross;
-import org.apache.pig.impl.logicalLayer.LODistinct;
-import org.apache.pig.impl.logicalLayer.LOFilter;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LOSort;
-import org.apache.pig.impl.logicalLayer.LOSplit;
-import org.apache.pig.impl.logicalLayer.LOUnion;
-import org.apache.pig.impl.logicalLayer.LOStore;
-import org.apache.pig.impl.logicalLayer.LOVisitor;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOJoin;
+import org.apache.pig.newplan.logical.relational.LOCross;
+import org.apache.pig.newplan.logical.relational.LODistinct;
+import org.apache.pig.newplan.logical.relational.LOFilter;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LOSort;
+import org.apache.pig.newplan.logical.relational.LOSplit;
+import org.apache.pig.newplan.logical.relational.LOUnion;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.IdentityHashSet;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.io.FileSpec;
@@ -63,18 +64,18 @@ import org.apache.pig.pen.util.MetricEva
import org.apache.pig.pen.util.PreOrderDepthFirstWalker;
import org.apache.pig.pen.util.ExampleTuple;
-public class LineageTrimmingVisitor extends LOVisitor {
+public class LineageTrimmingVisitor extends LogicalRelationalNodesVisitor {
LogicalPlan plan = null;
Map<LOLoad, DataBag> baseData;
Map<FileSpec, DataBag> inputToDataMap;
- Map<LogicalOperator, PhysicalOperator> LogToPhyMap = null;
+ Map<Operator, PhysicalOperator> LogToPhyMap = null;
PhysicalPlan physPlan = null;
double completeness = 100.0;
Log log = LogFactory.getLog(getClass());
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> AffinityGroups = new HashMap<LogicalOperator, Collection<IdentityHashSet<Tuple>>>();
- Map<LogicalOperator, LineageTracer> Lineage = new HashMap<LogicalOperator, LineageTracer>();
+ Map<Operator, Collection<IdentityHashSet<Tuple>>> AffinityGroups = new HashMap<Operator, Collection<IdentityHashSet<Tuple>>>();
+ Map<Operator, LineageTracer> Lineage = new HashMap<Operator, LineageTracer>();
boolean continueTrimming;
PigContext pc;
@@ -83,10 +84,9 @@ public class LineageTrimmingVisitor exte
public LineageTrimmingVisitor(LogicalPlan plan,
Map<LOLoad, DataBag> baseData,
ExampleGenerator eg,
- Map<LogicalOperator, PhysicalOperator> LogToPhyMap,
+ Map<Operator, PhysicalOperator> LogToPhyMap,
PhysicalPlan physPlan, PigContext pc) throws IOException, InterruptedException {
- super(plan, new PreOrderDepthFirstWalker<LogicalOperator, LogicalPlan>(
- plan));
+ super(plan, new PreOrderDepthFirstWalker(plan));
// this.baseData.putAll(baseData);
this.baseData = baseData;
this.plan = plan;
@@ -100,11 +100,11 @@ public class LineageTrimmingVisitor exte
public void init() throws IOException, InterruptedException {
- Map<LogicalOperator, DataBag> data = eg.getData();
+ Map<Operator, DataBag> data = eg.getData();
LineageTracer lineage = eg.getLineage();
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
- for (LogicalOperator leaf : plan.getLeaves()) {
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OpToEqClasses = eg.getLoToEqClassMap();
+ for (Operator leaf : plan.getSinks()) {
Lineage.put(leaf, lineage);
AffinityGroups.put(leaf, eg.getEqClasses());
}
@@ -116,7 +116,7 @@ public class LineageTrimmingVisitor exte
}
@Override
- protected void visit(LOCogroup cg) throws VisitorException {
+ public void visit(LOCogroup cg) throws FrontendException {
// can't separate CoGroup from succeeding ForEach
if (plan.getSuccessors(cg) != null && plan.getSuccessors(cg).get(0) instanceof LOForEach)
return;
@@ -128,91 +128,91 @@ public class LineageTrimmingVisitor exte
LineageTracer lineage = null;
// create affinity groups
- if (cg.getInputs().size() == 1) {
+ if (cg.getInputs(plan).size() == 1) {
lineage = eg.getLineage();
- AffinityGroups.put(cg.getInputs().get(0), eg.getEqClasses());
- Lineage.put(cg.getInputs().get(0), lineage);
+ AffinityGroups.put(cg.getInputs(plan).get(0), eg.getEqClasses());
+ Lineage.put(cg.getInputs(plan).get(0), lineage);
} else {
- for (LogicalOperator input : cg.getInputs()) {
+ for (Operator input : cg.getInputs(plan)) {
Lineage.put(input, eg.getLineage());
AffinityGroups.put(input, eg.getEqClasses());
}
}
} catch (Exception e) {
- throw new VisitorException("Exception : "+e.getMessage());
+ throw new FrontendException("Exception : "+e.getMessage());
}
}
}
@Override
- protected void visit(LOJoin join) throws VisitorException {
+ public void visit(LOJoin join) throws FrontendException {
if (continueTrimming) {
processOperator(join);
}
}
@Override
- protected void visit(LOCross cs) throws VisitorException {
+ public void visit(LOCross cs) throws FrontendException {
if(continueTrimming)
processOperator(cs);
}
@Override
- protected void visit(LODistinct dt) throws VisitorException {
+ public void visit(LODistinct dt) throws FrontendException {
if(continueTrimming)
processOperator(dt);
}
@Override
- protected void visit(LOFilter filter) throws VisitorException {
+ public void visit(LOFilter filter) throws FrontendException {
if (continueTrimming)
processOperator(filter);
}
@Override
- protected void visit(LOStore store) throws VisitorException {
+ public void visit(LOStore store) throws FrontendException {
if (continueTrimming)
processOperator(store);
}
@Override
- protected void visit(LOForEach forEach) throws VisitorException {
+ public void visit(LOForEach forEach) throws FrontendException {
if (continueTrimming)
processOperator(forEach);
}
@Override
- protected void visit(LOLimit limOp) throws VisitorException {
+ public void visit(LOLimit limOp) throws FrontendException {
if(continueTrimming)
processOperator(limOp);
}
@Override
- protected void visit(LOLoad load) throws VisitorException {
+ public void visit(LOLoad load) throws FrontendException {
if (continueTrimming)
processOperator(load);
}
@Override
- protected void visit(LOSort s) throws VisitorException {
+ public void visit(LOSort s) throws FrontendException {
if(continueTrimming)
processOperator(s);
}
@Override
- protected void visit(LOSplit split) throws VisitorException {
+ public void visit(LOSplit split) throws FrontendException {
if(continueTrimming)
processOperator(split);
}
@Override
- protected void visit(LOUnion u) throws VisitorException {
+ public void visit(LOUnion u) throws FrontendException {
if(continueTrimming)
processOperator(u);
@@ -322,6 +322,7 @@ public class LineageTrimmingVisitor exte
for (Tuple t : members)
tuplesToRetain.add(t);
}
+
Map<LOLoad, DataBag> newBaseData = new HashMap<LOLoad, DataBag>();
for (LOLoad loadOp : baseData.keySet()) {
DataBag data = baseData.get(loadOp);
@@ -338,10 +339,10 @@ public class LineageTrimmingVisitor exte
return newBaseData;
}
- private void processLoad(LOLoad ld) throws VisitorException {
+ private void processLoad(LOLoad ld) throws FrontendException {
// prune base records
- if (inputToDataMap.get(ld.getInputFile()) != null) {
- baseData.put(ld, inputToDataMap.get(ld.getInputFile()));
+ if (inputToDataMap.get(ld.getFileSpec()) != null) {
+ baseData.put(ld, inputToDataMap.get(ld.getFileSpec()));
return;
}
@@ -363,7 +364,7 @@ public class LineageTrimmingVisitor exte
newBaseData.put(ld, newData);
for (Map.Entry<LOLoad, DataBag> entry : baseData.entrySet()) {
if (entry.getKey() != ld) {
- if (!entry.getKey().getInputFile().equals(ld.getInputFile()))
+ if (!entry.getKey().getFileSpec().equals(ld.getFileSpec()))
newBaseData.put(entry.getKey(), entry.getValue());
else
newBaseData.put(entry.getKey(), newData);
@@ -373,10 +374,10 @@ public class LineageTrimmingVisitor exte
if (checkNewBaseData(newData, newBaseData, realData))
checkNewBaseData(newData, newBaseData, syntheticData);
- inputToDataMap.put(ld.getInputFile(), baseData.get(ld));
+ inputToDataMap.put(ld.getFileSpec(), baseData.get(ld));
}
- private boolean checkNewBaseData(DataBag data, Map<LOLoad, DataBag> newBaseData, Set<Tuple> loadData) throws VisitorException {
+ private boolean checkNewBaseData(DataBag data, Map<LOLoad, DataBag> newBaseData, Set<Tuple> loadData) throws FrontendException {
List<Pair<Tuple, Double>> sortedBase = new LinkedList<Pair<Tuple, Double>>();
DataBag oldData = BagFactory.getInstance().newDefaultBag();
oldData.addAll(data);
@@ -384,11 +385,11 @@ public class LineageTrimmingVisitor exte
for (Tuple t : loadData) {
data.add(t);
// obtain the derived data
- Map<LogicalOperator, DataBag> derivedData;
+ Map<Operator, DataBag> derivedData;
try {
derivedData = eg.getData(newBaseData);
} catch (Exception e) {
- throw new VisitorException("Exception: "+e.getMessage());
+ throw new FrontendException("Exception: "+e.getMessage());
}
double newCompleteness = MetricEvaluation.getCompleteness(null,
derivedData, eg.getLoToEqClassMap(), true);
@@ -412,11 +413,11 @@ public class LineageTrimmingVisitor exte
for (Pair<Tuple, Double> p : sortedBase) {
data.add(p.first);
// obtain the derived data
- Map<LogicalOperator, DataBag> derivedData;
+ Map<Operator, DataBag> derivedData;
try {
derivedData = eg.getData(newBaseData);
} catch (Exception e) {
- throw new VisitorException("Exception: "+e.getMessage());
+ throw new FrontendException("Exception: "+e.getMessage());
}
double newCompleteness = MetricEvaluation.getCompleteness(null,
derivedData, eg.getLoToEqClassMap(), true);
@@ -430,7 +431,7 @@ public class LineageTrimmingVisitor exte
return true;
}
- private void processOperator(LogicalOperator op) throws VisitorException {
+ private void processOperator(LogicalRelationalOperator op) throws FrontendException {
try {
if (op instanceof LOLoad) {
@@ -446,28 +447,28 @@ public class LineageTrimmingVisitor exte
if (continueTrimming == false)
return;
- LogicalOperator childOp = plan.getPredecessors(op).get(0);
+ Operator childOp = plan.getPredecessors(op).get(0);
if (op instanceof LOForEach && childOp instanceof LOCogroup)
{
LOCogroup cg = (LOCogroup) childOp;
- for (LogicalOperator input : cg.getInputs()) {
+ for (Operator input : cg.getInputs(plan)) {
AffinityGroups.put(input, eg.getEqClasses());
Lineage.put(input, eg.getLineage());
}
} else {
- List<LogicalOperator> childOps = plan.getPredecessors(op);
- for (LogicalOperator lo : childOps) {
+ List<Operator> childOps = plan.getPredecessors(op);
+ for (Operator lo : childOps) {
AffinityGroups.put(lo, eg.getEqClasses());
Lineage.put(lo, eg.getLineage());
}
}
} catch (Exception e) {
e.printStackTrace(System.out);
- throw new VisitorException("Exception: "+e.getMessage());
+ throw new FrontendException("Exception: "+e.getMessage());
}
}
- private boolean checkCompleteness(LogicalOperator op) throws Exception, VisitorException {
+ private boolean checkCompleteness(LogicalRelationalOperator op) throws Exception {
LineageTracer lineage = Lineage.get(op);
Lineage.remove(op);
@@ -479,7 +480,7 @@ public class LineageTrimmingVisitor exte
baseData, lineage, affinityGroups);
// obtain the derived data
- Map<LogicalOperator, DataBag> derivedData = eg.getData(newBaseData);
+ Map<Operator, DataBag> derivedData = eg.getData(newBaseData);
double newCompleteness = MetricEvaluation.getCompleteness(null,
derivedData, eg.getLoToEqClassMap(), true);
@@ -492,4 +493,8 @@ public class LineageTrimmingVisitor exte
return continueTrimming;
}
+
+ Map<LOLoad, DataBag> getBaseData() {
+ return baseData;
+ }
}
Modified: pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java (original)
+++ pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java Fri Dec 17 22:08:08 2010
@@ -56,8 +56,8 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.NullableTuple;
import org.apache.pig.impl.io.PigNullableWritable;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.VisitorException;
@@ -82,7 +82,6 @@ public class LocalMapReduceSimulator {
@SuppressWarnings("unchecked")
public void launchPig(PhysicalPlan php, Map<LOLoad, DataBag> baseData,
- Map<PhysicalOperator, LogicalOperator> poLoadToLogMap,
LineageTracer lineage,
IllustratorAttacher attacher,
ExampleGenerator eg,
@@ -158,7 +157,7 @@ public class LocalMapReduceSimulator {
for (POLoad ld : lds) {
input = output.get(ld.getLFile().getFileName());
if (input == null && baseData != null) {
- for (LogicalOperator lo : baseData.keySet()) {
+ for (LogicalRelationalOperator lo : baseData.keySet()) {
if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
{
input = baseData.get(lo);
@@ -174,7 +173,7 @@ public class LocalMapReduceSimulator {
input = output.get(ld.getLFile().getFileName());
if (input == null && baseData != null) {
if (input == null && baseData != null) {
- for (LogicalOperator lo : baseData.keySet()) {
+ for (LogicalRelationalOperator lo : baseData.keySet()) {
if (((LOLoad) lo).getSchemaFile().equals(ld.getLFile().getFileName()))
{
input = baseData.get(lo);
@@ -224,7 +223,9 @@ public class LocalMapReduceSimulator {
context = reduce.getIllustratorContext(job, intermediateData, (POPackage) pack);
reduce.run(context);
}
- phyToMRMap.putAll(mro.phyToMRMap);
+ for (PhysicalOperator key : mro.phyToMRMap.keySet())
+ for (PhysicalOperator value : mro.phyToMRMap.get(key))
+ phyToMRMap.put(key, value);
}
Modified: pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/DisplayExamples.java Fri Dec 17 22:08:08 2010
@@ -29,12 +29,14 @@ import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LOStore;
-import org.apache.pig.impl.logicalLayer.LOLimit;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LOLoad;
+import org.apache.pig.newplan.logical.relational.LOLimit;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.Operator;
import org.apache.pig.impl.util.IdentityHashSet;
//Class containing some generic printing methods to print example data in a simple/tabular form
@@ -44,9 +46,9 @@ public class DisplayExamples {
public static final int MAX_DATAATOM_LENGTH = 25;
static void printMetrics(
- LogicalOperator op,
- Map<LogicalOperator, DataBag> derivedData,
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses) {
+ Operator op,
+ Map<Operator, DataBag> derivedData,
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses) {
/*
* System.out.println("Realness : " + Metrics.getRealness(op,
* derivedData, true)); System.out.println("Completeness : " +
@@ -67,26 +69,34 @@ public class DisplayExamples {
}
public static String printTabular(LogicalPlan lp,
- Map<LogicalOperator, DataBag> exampleData,
- Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap) {
+ Map<Operator, DataBag> exampleData,
+ Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap) throws FrontendException {
StringBuffer output = new StringBuffer();
- Set<LogicalOperator> seen = new HashSet<LogicalOperator>();
- for (LogicalOperator currentOp : lp.getLeaves())
- printTabular(currentOp, exampleData, forEachInnerLogToDataMap, seen, output);
+ Set<Operator> seen = new HashSet<Operator>();
+ for (Operator currentOp : lp.getSinks()) {
+ if (currentOp instanceof LOStore && ((LOStore) currentOp).isTmpStore())
+ {
+ // display the branches of the temporary store first
+ printTabular(currentOp, lp, exampleData, forEachInnerLogToDataMap, seen, output);
+ }
+ }
+ for (Operator currentOp : lp.getSinks())
+ printTabular(currentOp, lp, exampleData, forEachInnerLogToDataMap, seen, output);
return output.toString();
}
- static void printTabular(LogicalOperator op,
- Map<LogicalOperator, DataBag> exampleData,
- Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap,
- Set<LogicalOperator> seen,
+ static void printTabular(Operator op,
+ LogicalPlan lp,
+ Map<Operator, DataBag> exampleData,
+ Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap,
+ Set<Operator> seen,
StringBuffer output) {
- List<LogicalOperator> inputs = op.getPlan().getPredecessors(op);
+ List<Operator> inputs = lp.getPredecessors(op);
if (inputs != null) { // to avoid an exception when op == LOLoad
- for (LogicalOperator Op : inputs) {
+ for (Operator Op : inputs) {
if (!seen.contains(Op))
- printTabular(Op, exampleData, forEachInnerLogToDataMap, seen, output);
+ printTabular(Op, lp, exampleData, forEachInnerLogToDataMap, seen, output);
}
}
seen.add(op);
@@ -95,8 +105,13 @@ public class DisplayExamples {
printNestedTabular((LOForEach)op, forEachInnerLogToDataMap, exampleData.get(op), output);
}
- if (op.getAlias() != null) {
+ if (((LogicalRelationalOperator)op).getAlias() != null) {
DataBag bag = exampleData.get(op);
+ if (op instanceof LOLoad && ((LOLoad)op).isCastInserted())
+ {
+ op = op.getPlan().getSuccessors(op).get(0);
+ bag = exampleData.get(op);
+ }
try {
DisplayTable(MakeArray(op, bag), op, bag, output);
} catch (FrontendException e) {
@@ -112,31 +127,29 @@ public class DisplayExamples {
// print out nested gen block in ForEach
static void printNestedTabular(LOForEach foreach,
- Map<LOForEach, Map<LogicalOperator, DataBag>> forEachInnerLogToDataMap,
+ Map<LOForEach, Map<LogicalRelationalOperator, DataBag>> forEachInnerLogToDataMap,
DataBag foreachData,
StringBuffer output) {
- List<LogicalPlan> plans = foreach.getForEachPlans();
- if (plans != null) {
- for (LogicalPlan plan : plans) {
- printNestedTabular(plan.getLeaves().get(0), foreach.getAlias(), foreachData, forEachInnerLogToDataMap.get(foreach), output);
- }
+ LogicalPlan plan = foreach.getInnerPlan();
+ if (plan != null) {
+ printNestedTabular(plan.getSinks().get(0), plan, foreach.getAlias(), foreachData, forEachInnerLogToDataMap.get(foreach), output);
}
}
- static void printNestedTabular(LogicalOperator lo, String foreachAlias, DataBag foreachData,
- Map<LogicalOperator, DataBag> logToDataMap, StringBuffer output) {
+ static void printNestedTabular(Operator lo, LogicalPlan lp, String foreachAlias, DataBag foreachData,
+ Map<LogicalRelationalOperator, DataBag> logToDataMap, StringBuffer output) {
- List<LogicalOperator> inputs = lo.getPlan().getPredecessors(lo);
+ List<Operator> inputs = lp.getPredecessors(lo);
if (inputs != null) {
- for (LogicalOperator op : inputs)
- printNestedTabular(op, foreachAlias, foreachData, logToDataMap, output);
+ for (Operator op : inputs)
+ printNestedTabular(op, lp, foreachAlias, foreachData, logToDataMap, output);
}
DataBag bag = logToDataMap.get(lo);
if (bag == null)
return;
- if (lo.getAlias() != null) {
+ if (((LogicalRelationalOperator)lo).getAlias() != null) {
try {
DisplayNestedTable(MakeArray(lo, bag), lo, foreachAlias, foreachData, bag, output);
} catch (FrontendException e) {
@@ -149,20 +162,20 @@ public class DisplayExamples {
}
}
- public static void printSimple(LogicalOperator op,
- Map<LogicalOperator, DataBag> exampleData) {
+ public static void printSimple(Operator op, LogicalPlan lp,
+ Map<Operator, DataBag> exampleData) {
DataBag bag = exampleData.get(op);
- List<LogicalOperator> inputs = op.getPlan().getPredecessors(op);
+ List<Operator> inputs = lp.getPredecessors(op);
if (inputs != null) {
- for (LogicalOperator lOp : inputs) {
- printSimple(lOp, exampleData);
+ for (Operator lOp : inputs) {
+ printSimple(lOp, lp, exampleData);
}
}
- if (op.getAlias() != null) {
+ if (((LogicalRelationalOperator)op).getAlias() != null) {
// printTable(op, bag, output);
// DisplayTable(MakeArray(op, bag), op, bag, output);
- System.out.println(op.getAlias() + " : " + bag);
+ System.out.println(((LogicalRelationalOperator)op).getAlias() + " : " + bag);
}
// System.out.println(op.getAlias() + " : " + bag);
}
@@ -178,13 +191,13 @@ public class DisplayExamples {
return str.toString();
}
- static void DisplayTable(String[][] table, LogicalOperator op, DataBag bag,
+ static void DisplayTable(String[][] table, Operator op, DataBag bag,
StringBuffer output) throws FrontendException {
if (op instanceof LOStore && ((LOStore) op).isTmpStore())
return;
- int cols = op.getSchema().getFields().size();
- List<FieldSchema> fields = op.getSchema().getFields();
+ int cols = ((LogicalRelationalOperator)op).getSchema().getFields().size();
+ List<LogicalSchema.LogicalFieldSchema> fields = ((LogicalRelationalOperator)op).getSchema().getFields();
int rows = (int) bag.size();
int[] maxColSizes = new int[cols];
for (int i = 0; i < cols; ++i) {
@@ -193,7 +206,8 @@ public class DisplayExamples {
maxColSizes[i] = 5;
}
int total = 0;
- int aliasLength = (op instanceof LOStore ? op.getAlias().length() + 12 : op.getAlias().length() + 4);
+ final String alias = ((LogicalRelationalOperator)op).getAlias();
+ int aliasLength = (op instanceof LOStore ? alias.length() + 12 : alias.length() + 4);
for (int j = 0; j < cols; ++j) {
for (int i = 0; i < rows; ++i) {
int length = table[i][j].length();
@@ -214,11 +228,11 @@ public class DisplayExamples {
false)
+ "\n");
if (op instanceof LOStore)
- output.append("| Store : " + op.getAlias() + AddSpaces(4, true) + " | ");
+ output.append("| Store : " + alias + AddSpaces(4, true) + " | ");
else
- output.append("| " + op.getAlias() + AddSpaces(4, true) + " | ");
+ output.append("| " + alias + AddSpaces(4, true) + " | ");
for (int i = 0; i < cols; ++i) {
- String field = fields.get(i).toString();
+ String field = fields.get(i).toString(false);
output.append(field
+ AddSpaces(maxColSizes[i] - field.length(), true) + " | ");
}
@@ -243,10 +257,11 @@ public class DisplayExamples {
+ "\n");
}
- static void DisplayNestedTable(String[][] table, LogicalOperator op, String foreachAlias, DataBag bag,
+ static void DisplayNestedTable(String[][] table, Operator op, String foreachAlias, DataBag bag,
DataBag foreachData, StringBuffer output) throws FrontendException {
- int cols = op.getSchema().getFields().size();
- List<FieldSchema> fields = op.getSchema().getFields();
+ LogicalRelationalOperator lop = (LogicalRelationalOperator) op;
+ int cols = lop.getSchema().getFields().size();
+ List<LogicalSchema.LogicalFieldSchema> fields = lop.getSchema().getFields();
int rows = (int) bag.size();
int[] maxColSizes = new int[cols];
for (int i = 0; i < cols; ++i) {
@@ -255,7 +270,8 @@ public class DisplayExamples {
maxColSizes[i] = 5;
}
int total = 0;
- int aliasLength = op.getAlias().length() + +foreachAlias.length() + 5;
+ final String alias = ((LogicalRelationalOperator)op).getAlias();
+ int aliasLength = alias.length() + +foreachAlias.length() + 5;
for (int j = 0; j < cols; ++j) {
for (int i = 0; i < rows; ++i) {
int length = table[i][j].length();
@@ -270,10 +286,10 @@ public class DisplayExamples {
.append(AddSpaces(total + 3 * (cols + 1) + aliasLength + 1,
false)
+ "\n");
- output.append("| " + foreachAlias + "." + op.getAlias() + AddSpaces(4, true) + " | ");
+ output.append("| " + foreachAlias + "." + alias + AddSpaces(4, true) + " | ");
for (int i = 0; i < cols; ++i) {
String field;
- field = fields.get(i).toString();
+ field = fields.get(i).toString(false);
output.append(field
+ AddSpaces(maxColSizes[i] - field.length(), true) + " | ");
}
@@ -298,10 +314,10 @@ public class DisplayExamples {
+ "\n");
}
- static String[][] MakeArray(LogicalOperator op, DataBag bag)
+ static String[][] MakeArray(Operator op, DataBag bag)
throws Exception {
int rows = (int) bag.size();
- int cols = op.getSchema().getFields().size();
+ int cols = ((LogicalRelationalOperator)op).getSchema().getFields().size();
String[][] table = new String[rows][cols];
Iterator<Tuple> it = bag.iterator();
for (int i = 0; i < rows; ++i) {
Modified: pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/MetricEvaluation.java Fri Dec 17 22:08:08 2010
@@ -24,19 +24,20 @@ import java.util.Map;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.impl.util.IdentityHashSet;
//Evaluates various metrics
public class MetricEvaluation {
- public static float getRealness(LogicalOperator op,
- Map<LogicalOperator, DataBag> exampleData, boolean overallRealness) {
+ public static float getRealness(Operator op,
+ Map<Operator, DataBag> exampleData, boolean overallRealness) {
// StringBuffer str = new StringBuffer();
int noTuples = 0;
int noSynthetic = 0;
- for (Map.Entry<LogicalOperator, DataBag> e : exampleData.entrySet()) {
+ for (Map.Entry<Operator, DataBag> e : exampleData.entrySet()) {
// if(e.getKey() instanceof LORead) continue;
- if (e.getKey().getAlias() == null)
+ if (((LogicalRelationalOperator)e.getKey()).getAlias() == null)
continue;
DataBag bag;
if (overallRealness) {
@@ -65,9 +66,9 @@ public class MetricEvaluation {
}
public static float getConciseness(
- LogicalOperator op,
- Map<LogicalOperator, DataBag> exampleData,
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
+ Operator op,
+ Map<Operator, DataBag> exampleData,
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
boolean overallConciseness) {
DataBag bag = exampleData.get(op);
@@ -85,7 +86,7 @@ public class MetricEvaluation {
conciseness = 0;
int noOperators = 0;
- for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
+ for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
.entrySet()) {
if (e.getKey().getAlias() == null)
continue;
@@ -108,9 +109,9 @@ public class MetricEvaluation {
}
public static float getCompleteness(
- LogicalOperator op,
- Map<LogicalOperator, DataBag> exampleData,
- Map<LogicalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
+ Operator op,
+ Map<Operator, DataBag> exampleData,
+ Map<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> OperatorToEqClasses,
boolean overallCompleteness) {
int noClasses = 0;
@@ -126,7 +127,7 @@ public class MetricEvaluation {
return 100 * ((float) noCoveredClasses) / (float) noClasses;
} else {
- for (Map.Entry<LogicalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
+ for (Map.Entry<LogicalRelationalOperator, Collection<IdentityHashSet<Tuple>>> e : OperatorToEqClasses
.entrySet()) {
noCoveredClasses = 0;
noClasses = 0;
Modified: pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java (original)
+++ pig/trunk/src/org/apache/pig/pen/util/PreOrderDepthFirstWalker.java Fri Dec 17 22:08:08 2010
@@ -23,16 +23,15 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.pig.impl.plan.DepthFirstWalker;
-import org.apache.pig.impl.plan.Operator;
-import org.apache.pig.impl.plan.OperatorPlan;
-import org.apache.pig.impl.plan.PlanVisitor;
-import org.apache.pig.impl.plan.PlanWalker;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.DepthFirstWalker;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.PlanVisitor;
+import org.apache.pig.newplan.PlanWalker;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.Utils;
-public class PreOrderDepthFirstWalker<O extends Operator, P extends OperatorPlan<O>>
- extends PlanWalker<O, P> {
+public class PreOrderDepthFirstWalker extends PlanWalker {
private boolean branchFlag = false;
@@ -40,7 +39,7 @@ public class PreOrderDepthFirstWalker<O
* @param plan
* Plan for this walker to traverse.
*/
- public PreOrderDepthFirstWalker(P plan) {
+ public PreOrderDepthFirstWalker(OperatorPlan plan) {
super(plan);
}
@@ -60,29 +59,28 @@ public class PreOrderDepthFirstWalker<O
* @throws VisitorException
* if an error is encountered while walking.
*/
- public void walk(PlanVisitor<O, P> visitor) throws VisitorException {
- List<O> leaves = mPlan.getLeaves();
- Set<O> seen = new HashSet<O>();
+ public void walk(PlanVisitor visitor) throws FrontendException {
+ List<Operator> leaves = plan.getSinks();
+ Set<Operator> seen = new HashSet<Operator>();
depthFirst(null, leaves, seen, visitor);
}
- public PlanWalker<O, P> spawnChildWalker(P plan) {
- return new DepthFirstWalker<O, P>(plan);
+ public PlanWalker spawnChildWalker(OperatorPlan plan) {
+ return new DepthFirstWalker(plan);
}
- @SuppressWarnings("unchecked")
- private void depthFirst(O node, Collection<O> predecessors, Set<O> seen,
- PlanVisitor<O, P> visitor) throws VisitorException {
+ private void depthFirst(Operator node, Collection<Operator> predecessors, Set<Operator> seen,
+ PlanVisitor visitor) throws FrontendException {
if (predecessors == null)
return;
boolean thisBranchFlag = branchFlag;
- for (O pred : predecessors) {
+ for (Operator pred : predecessors) {
if (seen.add(pred)) {
branchFlag = thisBranchFlag;
- pred.visit(visitor);
- Collection<O> newPredecessors = Utils.mergeCollection(mPlan.getPredecessors(pred), mPlan.getSoftLinkPredecessors(pred));
+ pred.accept(visitor);
+ Collection<Operator> newPredecessors = Utils.mergeCollection(plan.getPredecessors(pred), plan.getSoftLinkPredecessors(pred));
depthFirst(pred, newPredecessors, seen, visitor);
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestExampleGenerator.java Fri Dec 17 22:08:08 2010
@@ -32,7 +32,7 @@ import org.apache.pig.backend.executione
import org.apache.pig.data.DataBag;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.newplan.Operator;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -105,7 +105,7 @@ public class TestExampleGenerator extend
String query = "A = load " + A
+ " using PigStorage() as (x : int, y : int);\n";
pigserver.registerQuery(query);
- Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("A");
+ Map<Operator, DataBag> derivedData = pigserver.getExamples("A");
assertTrue(derivedData != null);
@@ -121,7 +121,7 @@ public class TestExampleGenerator extend
pigserver.registerQuery(query);
query = "B = filter A by x > 10;";
pigserver.registerQuery(query);
- Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigserver.getExamples("B");
assertTrue(derivedData != null);
@@ -137,7 +137,7 @@ public class TestExampleGenerator extend
pigserver.registerQuery(query);
query = "B = filter A by x > 5 AND y < 6;";
pigserver.registerQuery(query);
- Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigserver.getExamples("B");
assertTrue(derivedData != null);
}
@@ -158,7 +158,7 @@ public class TestExampleGenerator extend
pigserver.registerQuery(query);
query = "E = DISTINCT D;";
pigserver.registerQuery(query);
- Map<LogicalOperator, DataBag> derivedData = pigserver.getExamples("E");
+ Map<Operator, DataBag> derivedData = pigserver.getExamples("E");
assertTrue(derivedData != null);
@@ -172,7 +172,7 @@ public class TestExampleGenerator extend
+ " using PigStorage() as (x : int, y : int);");
pigServer.registerQuery("B = foreach A generate x + y as sum;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
assertTrue(derivedData != null);
}
@@ -185,7 +185,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("E = join A1 by x, B1 by x;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("E");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("E");
assertTrue(derivedData != null);
}
@@ -198,7 +198,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("E = join A1 by x, B1 by x;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("E");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("E");
assertTrue(derivedData != null);
}
@@ -210,7 +210,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A + " as (x, y);");
pigServer.registerQuery("B = load " + B + " as (x, y);");
pigServer.registerQuery("C = cogroup A by (x, y), B by (x, y);");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
}
@@ -221,7 +221,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A + " as (x, y);");
pigServer.registerQuery("B = load " + B + " as (x, y);");
pigServer.registerQuery("C = cogroup A by x, B by x;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
}
@@ -231,7 +231,7 @@ public class TestExampleGenerator extend
PigServer pigServer = new PigServer(pigContext);
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
pigServer.registerQuery("B = group A by x;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
assertTrue(derivedData != null);
@@ -243,7 +243,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
pigServer.registerQuery("B = group A by x;");
pigServer.registerQuery("C = foreach B generate group, COUNT(A);};");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
@@ -256,7 +256,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("B = FILTER A by x > 3;");
pigServer.registerQuery("C = group B by y;");
pigServer.registerQuery("D = foreach C generate group, COUNT(B);};");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("D");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("D");
assertTrue(derivedData != null);
@@ -269,7 +269,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("B = FILTER A by x > 3;");
pigServer.registerQuery("C = FILTER A by x < 3;");
pigServer.registerQuery("D = UNION B, C;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("D");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("D");
assertTrue(derivedData != null);
@@ -281,7 +281,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
pigServer.registerQuery("B = group A by x;");
pigServer.registerQuery("C = foreach B { FA = filter A by y == 6; generate group, COUNT(FA);};");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
@@ -293,7 +293,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A.toString() + " as (x:int, y:int);");
pigServer.registerQuery("B = group A by x;");
pigServer.registerQuery("C = foreach B { FA = filter A by y == 6; DA = DISTINCT FA; generate group, COUNT(DA);};");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
@@ -305,7 +305,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
pigServer.registerQuery("B = load " + B.toString() + " as (x, y);");
pigServer.registerQuery("C = union A, B;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
}
@@ -315,7 +315,7 @@ public class TestExampleGenerator extend
PigServer pigServer = new PigServer(pigContext);
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
pigServer.registerQuery("B = DISTINCT A;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
assertTrue(derivedData != null);
}
@@ -326,7 +326,7 @@ public class TestExampleGenerator extend
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
pigServer.registerQuery("B = load " + B.toString() + " as (x, y);");
pigServer.registerQuery("C = CROSS A, B;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("C");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("C");
assertTrue(derivedData != null);
}
@@ -336,7 +336,7 @@ public class TestExampleGenerator extend
PigServer pigServer = new PigServer(pigContext);
pigServer.registerQuery("A = load " + A.toString() + " as (x, y);");
pigServer.registerQuery("B = limit A 5;");
- Map<LogicalOperator, DataBag> derivedData = pigServer.getExamples("B");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("B");
assertTrue(derivedData != null);
}
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=1050503&r1=1050502&r2=1050503&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Fri Dec 17 22:08:08 2010
@@ -18,4 +18,4 @@ Reduce Plan Empty
| | |
| | Project[tuple][*] - scope-126
| |
- | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaabfkaaangfgogeepggebgmgmejgohahfhehihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlemaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaaeemaaamgnfdgpgghefegpefgeghgfhdhbaahoaaaeemaaaignfegpefgeghgfhdhbaahoaaaehihahdhcaacbgphcghcogbhagb
gdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphac
ogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhj
gfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdhfgmheheaabcemgkgbhggbcpgmg
bgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahi','','b','scope','true')) - scope-117
\ No newline at end of file
+ | |---b: Load(/tmp/input2:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaacfkaaangfgogeepggebgmgmejgohahfheemaaafgphagngbhaheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlhihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgoeaondlnechegpmlgacaaaiemaaakgneghcgpgnefgeghgfhdhbaahoaaademaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaognfdgpggheeghcgpgnefgeghgfhdhbaahoaaademaaamgnfdgpgghefegpefgeghgfhdhbaahoaaademaaaignfegpefgeghgfhdhbaahoaaad
hihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbhggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahhheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcg
bgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhdgjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaamfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgbgmgjgbhdhbaahoaaaoemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgo
gfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdh
fgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaabejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahahi','','b','scope','true')) - scope-117