You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/04/07 04:29:06 UTC
svn commit: r762596 [1/2] - in /hadoop/pig/branches/multiquery: ./
src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/
src/org/apache/pig/backend/...
Author: pradeepkth
Date: Tue Apr 7 02:29:05 2009
New Revision: 762596
URL: http://svn.apache.org/viewvc?rev=762596&view=rev
Log:
PIG-627: multiquery support incremental patch (hagleitn via pradeepkth)
Modified:
hadoop/pig/branches/multiquery/CHANGES.txt
hadoop/pig/branches/multiquery/src/org/apache/pig/PigServer.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MRPrinter.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/DotPOPrinter.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java
hadoop/pig/branches/multiquery/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java
hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LOStore.java
hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java
hadoop/pig/branches/multiquery/src/org/apache/pig/tools/grunt/GruntParser.java
hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMRCompiler.java
hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java
hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC5.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC9.gld
hadoop/pig/branches/multiquery/test/org/apache/pig/test/utils/GenPhyOp.java
Modified: hadoop/pig/branches/multiquery/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/CHANGES.txt?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/CHANGES.txt (original)
+++ hadoop/pig/branches/multiquery/CHANGES.txt Tue Apr 7 02:29:05 2009
@@ -595,3 +595,5 @@
to merge changes in trunk to multiquery branch (hagleitn via pradeepkth)
PIG-627: multiquery support incremental patch (hagleitn via pradeepkth)
+
+ PIG-627: multiquery support incremental patch (hagleitn via pradeepkth)
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/PigServer.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/PigServer.java Tue Apr 7 02:29:05 2009
@@ -36,6 +36,7 @@
import java.util.Set;
import java.util.Stack;
+import org.apache.pig.impl.plan.PlanException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
@@ -205,6 +206,20 @@
}
/**
+ * Returns whether there is anything to process in the current batch.
+ * @throws FrontendException
+ * @return true if there are no stores to process in the current
+ * batch, false otherwise.
+ */
+ public boolean isBatchEmpty() throws FrontendException {
+ if (currDAG == null) {
+ throw new IllegalStateException("setBatchOn() must be called first.");
+ }
+
+ return currDAG.isBatchEmpty();
+ }
+
+ /**
* Submits a batch of Pig commands for execution.
*
* @throws FrontendException
@@ -514,35 +529,7 @@
*/
public void explain(String alias,
PrintStream stream) throws IOException {
- explain(alias, "text", true, stream, stream, stream);
- }
-
- /**
- * Provide information on how a pig query will be executed.
- * @param alias Name of alias to explain.
- * @param format Format in which the explain should be printed
- * @param verbose Controls the amount of information printed
- * @param dir Directory to print the differnt plans into
- * @throws IOException if the requested alias cannot be found.
- */
- public void explain(String alias,
- String format,
- boolean verbose,
- String dir) throws IOException {
- try {
- PrintStream lps = new PrintStream(new File(dir,"logical_plan."+format));
- PrintStream pps = new PrintStream(new File(dir,"physical_plan."+format));
- PrintStream eps = new PrintStream(new File(dir,"exec_plan."+format));
- explain(alias, format, verbose, lps, pps, eps);
- lps.close();
- pps.close();
- eps.close();
-
- } catch (Exception e) {
- int errCode = 1067;
- String msg = "Unable to explain alias " + alias;
- throw new FrontendException(msg, errCode, PigException.INPUT, e);
- }
+ explain(alias, "text", true, false, stream, stream, stream);
}
/**
@@ -550,6 +537,9 @@
* @param alias Name of alias to explain.
* @param format Format in which the explain should be printed
* @param verbose Controls the amount of information printed
+ * @param markAsExecute When set will treat the explain like a
+ * call to execute in the respoect that all the pending stores are
+ * marked as complete.
* @param lps Stream to print the logical tree
* @param lps Stream to print the physical tree
* @param lps Stream to print the execution tree
@@ -558,6 +548,7 @@
public void explain(String alias,
String format,
boolean verbose,
+ boolean markAsExecute,
PrintStream lps,
PrintStream pps,
PrintStream eps) throws IOException {
@@ -574,6 +565,9 @@
lp.explain(lps, format, verbose);
pp.explain(pps, format, verbose);
pigContext.getExecutionEngine().explain(pp, eps, format, verbose);
+ if (markAsExecute) {
+ currDAG.markAsExecuted();
+ }
} catch (Exception e) {
int errCode = 1067;
String msg = "Unable to explain alias " + alias;
@@ -900,6 +894,8 @@
List<String> getScriptCache() { return scriptCache; }
boolean isBatchOn() { return batchMode; };
+
+ boolean isBatchEmpty() { return processedStores == storeOpTable.keySet().size(); }
void execute() throws ExecException, FrontendException {
pigContext.getProperties().setProperty(PigContext.JOB_NAME, PigContext.JOB_NAME_PREFIX + ":" + jobName);
@@ -907,6 +903,10 @@
processedStores = storeOpTable.keySet().size();
}
+ void markAsExecuted() {
+ processedStores = storeOpTable.keySet().size();
+ }
+
void setJobName(String name) {
jobName = name;
}
@@ -1032,16 +1032,27 @@
}
private void postProcess() throws IOException {
+
+ // Set the logical plan values correctly in all the operators
+ PlanSetter ps = new PlanSetter(lp);
+ ps.visit();
// The following code deals with store/load combination of
- // intermediate files. In this case we replace the load operator
- // with a (implicit) split operator.
+ // intermediate files. In this case we will replace the load operator
+ // with a (implicit) split operator, iff the load/store
+ // func is reversible (because that's when we can safely
+ // skip the load and keep going with the split output). If
+ // the load/store func is not reversible (or they are
+ // different functions), we connect the store and the load
+ // to remember the dependency.
for (LOLoad load : loadOps) {
for (LOStore store : storeOpTable.keySet()) {
String ifile = load.getInputFile().getFileName();
String ofile = store.getOutputFile().getFileName();
if (ofile.compareTo(ifile) == 0) {
- LogicalOperator storePred = lp.getPredecessors(store).get(0);
+ LoadFunc lFunc = (LoadFunc) pigContext.instantiateFuncFromSpec(load.getInputFile().getFuncSpec());
+ StoreFunc sFunc = (StoreFunc) pigContext.instantiateFuncFromSpec(store.getOutputFile().getFuncSpec());
+ if (lFunc.getClass() == sFunc.getClass() && lFunc instanceof ReversibleLoadStoreFunc) {
// In this case we remember the input file
// spec in the store. We might have to use it
@@ -1049,28 +1060,43 @@
// the store happens on a job boundary.
store.setInputSpec(load.getInputFile());
- lp.disconnect(store, load);
- lp.replace(load, storePred);
-
- List<LogicalOperator> succs = lp.getSuccessors(storePred);
-
- for (LogicalOperator succ : succs) {
- MultiMap<LogicalOperator, LogicalPlan> innerPls = null;
-
- // fix inner plans for cogroup and frjoin operators
- if (succ instanceof LOCogroup) {
- innerPls = ((LOCogroup)succ).getGroupByPlans();
- } else if (succ instanceof LOFRJoin) {
- innerPls = ((LOFRJoin)succ).getJoinColPlans();
- }
-
- if (innerPls != null) {
- if (innerPls.containsKey(load)) {
- Collection<LogicalPlan> pls = innerPls.get(load);
- innerPls.removeKey(load);
- innerPls.put(storePred, pls);
+ LogicalOperator storePred = lp.getPredecessors(store).get(0);
+
+ // In this case we remember the input file
+ // spec in the store. We might have to use it
+ // in the MR compiler to recreate the load, if
+ // the store happens on a job boundary.
+ store.setInputSpec(load.getInputFile());
+
+ lp.disconnect(store, load);
+ lp.replace(load, storePred);
+
+ List<LogicalOperator> succs = lp.getSuccessors(storePred);
+
+ for (LogicalOperator succ : succs) {
+ MultiMap<LogicalOperator, LogicalPlan> innerPls = null;
+
+ // fix inner plans for cogroup and frjoin operators
+ if (succ instanceof LOCogroup) {
+ innerPls = ((LOCogroup)succ).getGroupByPlans();
+ } else if (succ instanceof LOFRJoin) {
+ innerPls = ((LOFRJoin)succ).getJoinColPlans();
+ }
+
+ if (innerPls != null) {
+ if (innerPls.containsKey(load)) {
+ Collection<LogicalPlan> pls = innerPls.get(load);
+ innerPls.removeKey(load);
+ innerPls.put(storePred, pls);
+ }
}
}
+ } else {
+ try {
+ store.getPlan().connect(store, load);
+ } catch (PlanException ex) {
+ log.warn(ex.getMessage());
+ }
}
}
}
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Apr 7 02:29:05 2009
@@ -362,13 +362,9 @@
// set parent plan in all operators in map and reduce plans
// currently the parent plan is really used only when POStream is present in the plan
- PhysicalPlan[] plans = new PhysicalPlan[] { mro.mapPlan, mro.reducePlan };
- for (int i = 0; i < plans.length; i++) {
- for (Iterator<PhysicalOperator> it = plans[i].iterator(); it.hasNext();) {
- PhysicalOperator op = it.next();
- op.setParentPlan(plans[i]);
- }
- }
+ new PhyPlanSetter(mro.mapPlan).visit();
+ new PhyPlanSetter(mro.reducePlan).visit();
+
POPackage pack = null;
if(mro.reducePlan.isEmpty()){
//MapOnly Job
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Tue Apr 7 02:29:05 2009
@@ -76,6 +76,7 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.util.Pair;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper;
/**
* The compiler that compiles a given physical plan
@@ -128,6 +129,9 @@
//The output of compiling the inputs
MapReduceOper[] compiledInputs = null;
+
+ //Mapping of which MapReduceOper a store belongs to.
+ Map<POStore, MapReduceOper> storeToMapReduceMap;
//The split operators seen till now. If not
//maintained they will haunt you.
@@ -172,6 +176,7 @@
}
scope = roots.get(0).getOperatorKey().getScope();
messageCollector = new CompilationMessageCollector() ;
+ storeToMapReduceMap = new HashMap<POStore, MapReduceOper>();
}
public void randomizeFileLocalizer(){
@@ -219,8 +224,8 @@
}
}
- for (PhysicalOperator op : leaves) {
- POStore store = (POStore)op;
+ List<POStore> stores = PlanHelper.getStores(plan);
+ for (POStore store: stores) {
FileLocalizer.registerDeleteOnFail(store.getSFile().getFileName(), pigContext);
compile(store);
}
@@ -273,6 +278,41 @@
//op.
List<PhysicalOperator> predecessors = plan.getPredecessors(op);
if (predecessors != null && predecessors.size() > 0) {
+ // When processing an entire script (multiquery), we can
+ // get into a situation where a load has
+ // predecessors. This means that it depends on some store
+ // earlier in the plan. We need to take that dependency
+ // and connect the respective MR operators, while at the
+ // same time removing the connection between the Physical
+ // operators. That way the jobs will run in the right
+ // order.
+ if (op instanceof POLoad) {
+
+ if (predecessors.size() != 1) {
+ int errCode = 2035;
+ String msg = "Expected at most one predecessor of load. Got "+predecessors.size();
+ throw new PlanException(msg, errCode, PigException.BUG);
+ }
+
+ PhysicalOperator p = predecessors.get(0);
+ if (!(p instanceof POStore)) {
+ int errCode = 2036;
+ String msg = "Predecessor of load should be a store. Got "+p.getClass();
+ throw new PlanException(msg, errCode, PigException.BUG);
+ }
+
+ // Need new operator
+ curMROp = getMROp();
+ curMROp.mapPlan.add(op);
+ MRPlan.add(curMROp);
+
+ MapReduceOper oper = storeToMapReduceMap.get((POStore)p);
+
+ plan.disconnect(op, p);
+ MRPlan.connect(oper, curMROp);
+ return;
+ }
+
Collections.sort(predecessors);
compiledInputs = new MapReduceOper[predecessors.size()];
int i = -1;
@@ -639,6 +679,7 @@
public void visitStore(POStore op) throws VisitorException{
try{
+ storeToMapReduceMap.put(op, curMROp);
nonBlocking(op);
}catch(Exception e){
int errCode = 2034;
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Tue Apr 7 02:29:05 2009
@@ -192,6 +192,7 @@
DotMRPrinter printer =new DotMRPrinter(mrp, ps);
printer.setVerbose(verbose);
printer.dump();
+ ps.println("");
}
}
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MRPrinter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MRPrinter.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MRPrinter.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/plans/MRPrinter.java Tue Apr 7 02:29:05 2009
@@ -43,9 +43,9 @@
public MRPrinter(PrintStream ps, MROperPlan plan) {
super(plan, new DepthFirstWalker<MapReduceOper, MROperPlan>(plan));
mStream = ps;
- mStream.println("--------------------------------------------------");
- mStream.println("| Map Reduce Plan |");
- mStream.println("--------------------------------------------------");
+ mStream.println("#--------------------------------------------------");
+ mStream.println("# Map Reduce Plan ");
+ mStream.println("#--------------------------------------------------");
}
public void setVerbose(boolean verbose) {
@@ -81,6 +81,7 @@
mStream.println("Quantile file: " + mr.getQuantFile());
}
mStream.println("----------------");
+ mStream.println("");
}
}
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java Tue Apr 7 02:29:05 2009
@@ -1157,8 +1157,6 @@
@Override
public void visit(LOLoad loLoad) throws VisitorException {
String scope = loLoad.getOperatorKey().scope;
- // This would be a root operator. We don't need to worry about finding
- // its predecessors
POLoad load = new POLoad(new OperatorKey(scope, nodeGen
.getNextNodeId(scope)), loLoad.isSplittable());
load.setLFile(loLoad.getInputFile());
@@ -1167,6 +1165,23 @@
currentPlan.add(load);
LogToPhyMap.put(loLoad, load);
this.load = loLoad.getLoadFunc();
+
+ // Load is typically a root operator, but in the multiquery
+ // case it might have a store as a predecessor.
+ List<LogicalOperator> op = loLoad.getPlan().getPredecessors(loLoad);
+ PhysicalOperator from;
+
+ if(op != null) {
+ from = LogToPhyMap.get(op.get(0));
+
+ try {
+ currentPlan.connect(from, load);
+ } catch (PlanException e) {
+ int errCode = 2015;
+ String msg = "Invalid physical operators in the physical plan" ;
+ throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
+ }
+ }
}
@Override
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/DotPOPrinter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/DotPOPrinter.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/DotPOPrinter.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/DotPOPrinter.java Tue Apr 7 02:29:05 2009
@@ -65,7 +65,14 @@
protected String[] getAttributes(PhysicalOperator op) {
if (op instanceof POStore || op instanceof POLoad) {
String[] attributes = new String[3];
- attributes[0] = "label=\""+getName(op).replace(":",",\\n")+"\"";
+ String name = getName(op);
+ int idx = name.lastIndexOf(":");
+ if (idx != -1) {
+ String part1 = name.substring(0,idx);
+ String part2 = name.substring(idx+1,name.length());
+ name = part1+",\\n"+part2;
+ }
+ attributes[0] = "label=\""+name+"\"";
attributes[1] = "style=\"filled\"";
attributes[2] = "fillcolor=\"gray\"";
return attributes;
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java Tue Apr 7 02:29:05 2009
@@ -108,6 +108,7 @@
if (format.equals("text")) {
explain((OutputStream)ps, verbose);
+ ps.println("");
} else if (format.equals("dot")) {
DotPOPrinter pp = new DotPOPrinter(this, ps);
pp.setVerbose(verbose);
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java Tue Apr 7 02:29:05 2009
@@ -148,7 +148,7 @@
@Override
public boolean supportsMultipleOutputs() {
- return false;
+ return true;
}
@Override
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/util/PlanHelper.java Tue Apr 7 02:29:05 2009
@@ -26,6 +26,9 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.*;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -41,42 +44,35 @@
private PlanHelper() {}
/**
- * Get all the store operators in the plan
+ * Get all the store operators in the plan in the right dependency order
* @param plan
* @return List of stores (could be empty)
*/
public static List<POStore> getStores(PhysicalPlan plan) {
- List<POStore> stores = new LinkedList<POStore>();
- List<PhysicalOperator> leaves = plan.getLeaves();
- for (PhysicalOperator leaf: leaves) {
- if (leaf instanceof POStore) {
- stores.add((POStore)leaf);
- }
- if (leaf instanceof POSplit) {
- List<PhysicalPlan> pls = ((POSplit)leaf).getPlans();
- for (PhysicalPlan pl : pls) {
- List<POStore> nestedStores = getStores(pl);
- stores.addAll(nestedStores);
- }
- }
+ LoadStoreFinder finder = new LoadStoreFinder(plan);
+
+ try {
+ finder.visit();
+ } catch (VisitorException ve) {
+ log.warn("Exception in getStores(): "+ve.getMessage());
}
- return stores;
+ return finder.getStores();
}
/**
- * Get all the load operators in the plan
+ * Get all the load operators in the plan in the right dependency order
* @param plan
* @return List of loads (could be empty)
*/
public static List<POLoad> getLoads(PhysicalPlan plan) {
- List<POLoad> loads = new LinkedList<POLoad>();
- List<PhysicalOperator> roots = plan.getRoots();
- for (PhysicalOperator root: roots) {
- if (root instanceof POLoad) {
- loads.add((POLoad)root);
- }
+ LoadStoreFinder finder = new LoadStoreFinder(plan);
+
+ try {
+ finder.visit();
+ } catch (VisitorException ve) {
+ log.warn("Exception in getLoads(): "+ve.getMessage());
}
- return loads;
+ return finder.getLoads();
}
/**
@@ -95,4 +91,40 @@
return new Path("rel/"+pathStr).toString();
}
}
+
+ private static class LoadStoreFinder extends PhyPlanVisitor {
+ private List<POLoad> loads;
+ private List<POStore> stores;
+
+ LoadStoreFinder(PhysicalPlan plan) {
+ super(plan, new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(plan));
+ stores = new LinkedList<POStore>();
+ loads = new LinkedList<POLoad>();
+ }
+
+ @Override
+ public void visit() throws VisitorException {
+ super.visit();
+ }
+
+ @Override
+ public void visitStore(POStore st) throws VisitorException {
+ super.visitStore(st);
+ stores.add(st);
+ }
+
+ @Override
+ public void visitLoad(POLoad load) throws VisitorException {
+ super.visitLoad(load);
+ loads.add(load);
+ }
+
+ public List<POStore> getStores() {
+ return stores;
+ }
+
+ public List<POLoad> getLoads() {
+ return loads;
+ }
+ }
}
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/backend/local/executionengine/LocalPigLauncher.java Tue Apr 7 02:29:05 2009
@@ -21,7 +21,9 @@
import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
+import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -72,8 +74,32 @@
op.setStoreImpl(new LocalPOStoreImpl(pc));
op.setUp();
}
+
+ // We need to handle stores that have loads as successors
+ // first. PlanHelper's getStores has returned those in the
+ // dependency order, so that's how we will run them.
+ for (Iterator<POStore> it = stores.iterator(); it.hasNext(); ) {
+ POStore op = it.next();
+
+ List<PhysicalOperator> sucs = new ArrayList<PhysicalOperator>();
+ if (php.getSuccessors(op) != null) {
+ sucs.addAll(php.getSuccessors(op));
+ }
+
+ if (sucs.size() != 0) {
+ log.info("running store with dependencies");
+ POStore[] st = new POStore[1];
+ st[0] = op;
+ failedJobs += runPipeline(st);
+ for (PhysicalOperator suc: sucs) {
+ php.disconnect(op, suc);
+ }
+ it.remove();
+ }
+ }
- failedJobs = runPipeline(stores.toArray(new POStore[0]));
+ // The remaining stores can be run together.
+ failedJobs += runPipeline(stores.toArray(new POStore[0]));
UDFFinishVisitor finisher = new UDFFinishVisitor(php, new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(php));
finisher.visit();
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LOStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LOStore.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LOStore.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LOStore.java Tue Apr 7 02:29:05 2009
@@ -112,7 +112,7 @@
@Override
public boolean supportsMultipleOutputs() {
- return false;
+ return true;
}
public void visit(LOVisitor v) throws VisitorException {
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Tue Apr 7 02:29:05 2009
@@ -73,8 +73,8 @@
DotLOPrinter lpp = new DotLOPrinter(this, ps);
lpp.setVerbose(verbose);
lpp.dump();
+ ps.println("");
}
- ps.println("");
}
// public String toString() {
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/impl/logicalLayer/optimizer/ImplicitSplitInserter.java Tue Apr 7 02:29:05 2009
@@ -28,6 +28,7 @@
import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LOSplit;
+import org.apache.pig.impl.logicalLayer.LOStore;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
@@ -53,6 +54,7 @@
List<LogicalOperator> succs = mPlan.getSuccessors(op);
if (succs == null || succs.size() < 2) return false;
if (op instanceof LOSplit) return false;
+ if (op instanceof LOStore) return false;
return true;
} catch (Exception e) {
int errCode = 2048;
Modified: hadoop/pig/branches/multiquery/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/src/org/apache/pig/tools/grunt/GruntParser.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/branches/multiquery/src/org/apache/pig/tools/grunt/GruntParser.java Tue Apr 7 02:29:05 2009
@@ -37,6 +37,7 @@
import java.util.List;
import java.util.ArrayList;
import java.util.Properties;
+import java.util.Date;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.PrintStream;
@@ -92,6 +93,7 @@
private void init() {
mDone = false;
mLoadOnly = false;
+ mExplain = null;
}
private void setBatchOn() {
@@ -99,8 +101,14 @@
}
private void executeBatch() throws IOException {
- if (mPigServer.isBatchOn() && !mLoadOnly) {
- mPigServer.executeBatch();
+ if (mPigServer.isBatchOn()) {
+ if (mExplain != null) {
+ explainCurrentBatch();
+ }
+
+ if (!mLoadOnly) {
+ mPigServer.executeBatch();
+ }
}
}
@@ -196,52 +204,83 @@
String format, String target,
List<String> params, List<String> files)
throws IOException, ParseException {
+
+ if (null != mExplain) {
+ return;
+ }
- PrintStream out = System.out;
-
- if (script != null) {
- setBatchOn();
- try {
- loadScript(script, true, true, params, files);
- } catch(IOException e) {
- discardBatch();
- throw e;
+ try {
+ mExplain = new ExplainState(alias, target, script, isVerbose, format);
+
+ if (script != null) {
+ if (!"true".equalsIgnoreCase(mPigServer.
+ getPigContext()
+ .getProperties().
+ getProperty("opt.multiquery","true"))) {
+ throw new ParseException("Cannot explain script if multiquery is disabled.");
+ }
+ setBatchOn();
+ try {
+ loadScript(script, true, true, params, files);
+ } catch(IOException e) {
+ discardBatch();
+ throw e;
} catch (ParseException e) {
+ discardBatch();
+ throw e;
+ }
+ }
+
+ mExplain.mLast = true;
+ explainCurrentBatch();
+
+ } finally {
+ if (script != null) {
discardBatch();
- throw e;
}
+ mExplain = null;
}
+ }
+
+ protected void explainCurrentBatch() throws IOException {
+ PrintStream lp = System.out;
+ PrintStream pp = System.out;
+ PrintStream ep = System.out;
- if (target != null) {
- File file = new File(target);
+ if (!(mExplain.mLast && mExplain.mCount == 0)) {
+ if (mPigServer.isBatchEmpty()) {
+ return;
+ }
+ }
+ mExplain.mCount++;
+ boolean markAsExecuted = (mExplain.mScript != null);
+
+ if (mExplain.mTarget != null) {
+ File file = new File(mExplain.mTarget);
+
if (file.isDirectory()) {
- mPigServer.explain(alias, format, isVerbose, target);
- if (script != null) {
- discardBatch();
- }
- return;
+ String sCount = (mExplain.mLast && mExplain.mCount == 1)?"":"_"+mExplain.mCount;
+ lp = new PrintStream(new File(file, "logical_plan-"+mExplain.mTime+sCount+"."+mExplain.mFormat));
+ pp = new PrintStream(new File(file, "physical_plan-"+mExplain.mTime+sCount+"."+mExplain.mFormat));
+ ep = new PrintStream(new File(file, "exec_plan-"+mExplain.mTime+sCount+"."+mExplain.mFormat));
+ mPigServer.explain(mExplain.mAlias, mExplain.mFormat,
+ mExplain.mVerbose, markAsExecuted, lp, pp, ep);
+ lp.close();
+ pp.close();
+ ep.close();
}
else {
- try {
- out = new PrintStream(new FileOutputStream(target));
- }
- catch (FileNotFoundException fnfe) {
- if (script != null) {
- discardBatch();
- }
- throw new ParseException("File not found: " + target);
- } catch (SecurityException se) {
- if (script != null) {
- discardBatch();
- }
- throw new ParseException("Cannot access file: " + target);
- }
+ boolean append = !(mExplain.mCount==1);
+ lp = pp = ep = new PrintStream(new FileOutputStream(mExplain.mTarget, append));
+ mPigServer.explain(mExplain.mAlias, mExplain.mFormat,
+ mExplain.mVerbose, markAsExecuted, lp, pp, ep);
+ lp.close();
}
}
- mPigServer.explain(alias, format, isVerbose, out, out, out);
- if (script != null) {
- discardBatch();
+ else {
+ mPigServer.explain(mExplain.mAlias, mExplain.mFormat,
+ mExplain.mVerbose, markAsExecuted, lp, pp, ep);
}
}
@@ -331,6 +370,7 @@
parser.setConsoleReader(reader);
parser.setInteractive(interactive);
parser.setLoadOnly(loadOnly);
+ parser.mExplain = mExplain;
parser.prompt();
while(!parser.isDone()) {
@@ -652,6 +692,29 @@
}
}
+ private class ExplainState {
+ public long mTime;
+ public int mCount;
+ public String mAlias;
+ public String mTarget;
+ public String mScript;
+ public boolean mVerbose;
+ public String mFormat;
+ public boolean mLast;
+
+ public ExplainState(String alias, String target, String script,
+ boolean verbose, String format) {
+ mTime = new Date().getTime();
+ mCount = 0;
+ mAlias = alias;
+ mTarget = target;
+ mScript = script;
+ mVerbose = verbose;
+ mFormat = format;
+ mLast = false;
+ }
+ }
+
private PigServer mPigServer;
private DataStorage mDfs;
private DataStorage mLfs;
@@ -659,5 +722,5 @@
private JobClient mJobClient;
private boolean mDone;
private boolean mLoadOnly;
-
+ private ExplainState mExplain;
}
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMRCompiler.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMRCompiler.java Tue Apr 7 02:29:05 2009
@@ -98,7 +98,7 @@
// WILL OVERWRITE THE GOLDEN FILES - So use this
// with caution and only for the testcases you need
// and are sure of
- private boolean generate = true;
+ private boolean generate = false;
@Before
public void setUp() throws ExecException {
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java Tue Apr 7 02:29:05 2009
@@ -1135,6 +1135,45 @@
}
}
+ @Test
+ public void testStoreOrder() {
+ System.out.println("===== multi-query store order =====");
+
+ try {
+ myPig.setBatchOn();
+ myPig.registerQuery("a = load 'file:test/org/apache/pig/test/data/passwd';");
+ myPig.registerQuery("store a into '/tmp/output1' using BinStorage();");
+ myPig.registerQuery("a = load '/tmp/output1';");
+ myPig.registerQuery("store a into '/tmp/output2';");
+ myPig.registerQuery("a = load '/tmp/output1';");
+ myPig.registerQuery("store a into '/tmp/output3';");
+ myPig.registerQuery("a = load '/tmp/output2' using BinStorage();");
+ myPig.registerQuery("store a into '/tmp/output4';");
+ myPig.registerQuery("a = load '/tmp/output2';");
+ myPig.registerQuery("b = load '/tmp/output1';");
+ myPig.registerQuery("c = cogroup a by $0, b by $0;");
+ myPig.registerQuery("store c into '/tmp/output5';");
+
+ LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 17);
+ MROperPlan mp = checkMRPlan(pp, 1, 3, 5);
+
+ myPig.executeBatch();
+ myPig.discardBatch();
+
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output1"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output2"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output3"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output4"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output5"));
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
// --------------------------------------------------------------------------
// Helper methods
@@ -1284,6 +1323,7 @@
FileLocalizer.delete("/tmp/output2", myPig.getPigContext());
FileLocalizer.delete("/tmp/output3", myPig.getPigContext());
FileLocalizer.delete("/tmp/output4", myPig.getPigContext());
+ FileLocalizer.delete("/tmp/output5", myPig.getPigContext());
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java Tue Apr 7 02:29:05 2009
@@ -19,6 +19,7 @@
import java.io.StringReader;
import java.io.IOException;
+import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Properties;
@@ -437,6 +438,46 @@
}
}
+ @Test
+ public void testStoreOrder() {
+ System.out.println("===== multi-query store order =====");
+
+ try {
+ myPig.setBatchOn();
+ myPig.registerQuery("a = load 'file:test/org/apache/pig/test/data/passwd';");
+ myPig.registerQuery("store a into '/tmp/output1' using BinStorage();");
+ myPig.registerQuery("a = load '/tmp/output1';");
+ myPig.registerQuery("store a into '/tmp/output2';");
+ myPig.registerQuery("a = load '/tmp/output1';");
+ myPig.registerQuery("store a into '/tmp/output3';");
+ myPig.registerQuery("a = load '/tmp/output2' using BinStorage();");
+ myPig.registerQuery("store a into '/tmp/output4';");
+ myPig.registerQuery("a = load '/tmp/output2';");
+ myPig.registerQuery("b = load '/tmp/output1';");
+ myPig.registerQuery("c = cogroup a by $0, b by $0;");
+ myPig.registerQuery("store c into '/tmp/output5';");
+
+ LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 16);
+
+ myPig.executeBatch();
+ myPig.discardBatch();
+
+ Assert.assertTrue(new File("/tmp/output1").exists());
+ Assert.assertTrue(new File("/tmp/output2").exists());
+ Assert.assertTrue(new File("/tmp/output3").exists());
+ Assert.assertTrue(new File("/tmp/output4").exists());
+ Assert.assertTrue(new File("/tmp/output5").exists());
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ } finally {
+ deleteOutputFiles();
+ }
+ }
+
// --------------------------------------------------------------------------
// Helper methods
@@ -523,6 +564,8 @@
FileLocalizer.delete("/tmp/output1", myPig.getPigContext());
FileLocalizer.delete("/tmp/output2", myPig.getPigContext());
FileLocalizer.delete("/tmp/output3", myPig.getPigContext());
+ FileLocalizer.delete("/tmp/output4", myPig.getPigContext());
+ FileLocalizer.delete("/tmp/output5", myPig.getPigContext());
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld Tue Apr 7 02:29:05 2009
@@ -1,17 +1,19 @@
MapReduce(-1) - -78:
-| Store(DummyFil:DummyLdr) - --7868505214447593853
+| Store(DummyFil:DummyLdr) - -5515425171581967372
| |
-| |---Filter[tuple] - --5177745552827005198
+| |---Filter[tuple] - --6061281703859425960
+| | |
+| | Constant(true) - --3396897091865664764
| |
-| |---Package[tuple]{Unknown} - --6430355428631435461
-| Local Rearrange[tuple]{Unknown}(false) - -8729990799028586395
+| |---Package[tuple]{Unknown} - --5758282087831209061
+| Local Rearrange[tuple]{Unknown}(false) - -3709512757404691843
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -77
|
|---MapReduce(-1) - -75:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -76
| |
- | |---Package[tuple]{Unknown} - -4721502244557927278
- | Local Rearrange[tuple]{Unknown}(false) - --7681398237172009051
+ | |---Package[tuple]{Unknown} - --2057425961601007773
+ | Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
| |
- | |---Load(DummyFil:DummyLdr) - -6620645493024302760
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -7506868571066332964
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld Tue Apr 7 02:29:05 2009
@@ -1,8 +1,8 @@
MapReduce(-1) - -11:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - --2449990780759860228
+| Store(DummyFil:DummyLdr) - -7490898804471997380
| |
-| |---Union[tuple] - -4061122832660258194
+| |---Union[tuple] - -3720949273928245639
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -12
| |
@@ -11,25 +11,29 @@
|---MapReduce(-1) - -2:
| | Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -13
| | |
-| | |---Filter[tuple] - --171729478481529687
+| | |---Filter[tuple] - --2449990780759860228
+| | | |
+| | | Constant(true) - --8248982303554009
| | |
-| | |---Package[tuple]{Unknown} - -3527883492192621891
+| | |---Package[tuple]{Unknown} - -4061122832660258194
| | Union[tuple] - -3
| | |
-| | |---Local Rearrange[tuple]{Unknown}(false) - --2655303127943013956
+| | |---Local Rearrange[tuple]{Unknown}(false) - -3527883492192621891
| | | |
-| | | |---Load(DummyFil:DummyLdr) - --3833933141637499382
+| | | |---Load(DummyFil:DummyLdr) - --6402314745592504008
| | |
-| | |---Local Rearrange[tuple]{Unknown}(false) - -7473175511145418837
+| | |---Local Rearrange[tuple]{Unknown}(false) - -8637487025682524492
| | |
-| | |---Filter[tuple] - --6402314745592504008
+| | |---Filter[tuple] - -7473175511145418837
+| | | |
+| | | Constant(true) - --2655303127943013956
| | |
-| | |---Load(DummyFil:DummyLdr) - --838807233869503381
+| | |---Load(DummyFil:DummyLdr) - --3833933141637499382
|
|---MapReduce(-1) - -6:
| Store(file:/tmp/temp-1456742965/tmp774375955:org.apache.pig.builtin.BinStorage) - -15
| |
- | |---Package[tuple]{Unknown} - -990040854696137546
+ | |---Package[tuple]{Unknown} - -5679595123645092366
| Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -7
|
| Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -9
@@ -37,21 +41,23 @@
|---MapReduce(30) - -4:
| | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -8
| | |
- | | |---Local Rearrange[tuple]{Unknown}(false) - --5623550231721294978
+ | | |---Local Rearrange[tuple]{Unknown}(false) - --8216215966586363937
| | |
- | | |---Package[tuple]{Unknown} - --6259721534861268730
- | | Local Rearrange[tuple]{Unknown}(false) - --7212359720440714287
+ | | |---Package[tuple]{Unknown} - --7212359720440714287
+ | | Local Rearrange[tuple]{Unknown}(false) - -7469509242284658386
| | |
- | | |---Load(DummyFil:DummyLdr) - -6748240903696823165
+ | | |---Load(DummyFil:DummyLdr) - -990040854696137546
|
|---MapReduce(20) - -5:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -10
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - -5679595123645092366
+ | |---Local Rearrange[tuple]{Unknown}(false) - --5623550231721294978
| |
- | |---Package[tuple]{Unknown} - -8345455294066939854
- | Local Rearrange[tuple]{Unknown}(false) - -2043312794799763441
+ | |---Package[tuple]{Unknown} - --6259721534861268730
+ | Local Rearrange[tuple]{Unknown}(false) - -3248199015665744565
| |
| |---Filter[tuple] - -6520791719738296531
+ | | |
+ | | Constant(true) - -2043312794799763441
| |
| |---Load(DummyFil:DummyLdr) - --5314747545448923824
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld Tue Apr 7 02:29:05 2009
@@ -1,8 +1,8 @@
MapReduce(-1) - -26:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - -2288440041178982968
+| Store(DummyFil:DummyLdr) - -3900012572437255236
| |
-| |---Union[tuple] - -913150185705910016
+| |---Union[tuple] - -8760996681222683693
| |
| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -27
| |
@@ -11,39 +11,45 @@
|---MapReduce(-1) - -18:
| | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -28
| | |
-| | |---Filter[tuple] - -3055962796670893316
+| | |---Filter[tuple] - -956528893337238225
+| | | |
+| | | Constant(true) - -7391599663069134339
| | |
-| | |---Package[tuple]{Unknown} - -3943647700946858188
+| | |---Package[tuple]{Unknown} - -4024598850351165272
| | Union[tuple] - -19
| | |
-| | |---Local Rearrange[tuple]{Unknown}(false) - -2833954415250116776
+| | |---Local Rearrange[tuple]{Unknown}(false) - -913150185705910016
| | | |
-| | | |---Filter[tuple] - --4083686173685839766
+| | | |---Filter[tuple] - -3943647700946858188
+| | | | |
+| | | | Constant(true) - -3055962796670893316
| | | |
-| | | |---Load(DummyFil:DummyLdr) - --5733160635931065595
+| | | |---Load(DummyFil:DummyLdr) - -4962214768762054129
| | |
-| | |---Local Rearrange[tuple]{Unknown}(false) - -4962214768762054129
+| | |---Local Rearrange[tuple]{Unknown}(false) - -2288440041178982968
| | |
-| | |---Load(DummyFil:DummyLdr) - --1115934782004129477
+| | |---Load(DummyFil:DummyLdr) - -7577375165361647685
|
|---MapReduce(-1) - -22:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -30
| |
- | |---Package[tuple]{Unknown} - -6108697957385222190
+ | |---Package[tuple]{Unknown} - -3742910951635599848
| Union[tuple] - -23
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -24
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - -7490898804471997380
+ | |---Local Rearrange[tuple]{Unknown}(false) - -689137294940608050
| |
- | |---Filter[tuple] - -3720949273928245639
+ | |---Filter[tuple] - -6108697957385222190
+ | | |
+ | | Constant(true) - --8723304958939002625
| |
- | |---Load(DummyFil:DummyLdr) - --8248982303554009
+ | |---Load(DummyFil:DummyLdr) - --961974426096759446
|
|---MapReduce(-1) - -20:
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -25
| |
- | |---Package[tuple]{Unknown} - --8723304958939002625
- | Local Rearrange[tuple]{Unknown}(false) - -689137294940608050
+ | |---Package[tuple]{Unknown} - --5733160635931065595
+ | Local Rearrange[tuple]{Unknown}(false) - --1115934782004129477
| |
- | |---Load(DummyFil:DummyLdr) - -3742910951635599848
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -2833954415250116776
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld Tue Apr 7 02:29:05 2009
@@ -1,18 +1,22 @@
MapReduce(-1) - -37:
-| Store(DummyFil:DummyLdr) - -340188275237520979
+| Store(DummyFil:DummyLdr) - --1596982214453270401
| |
-| |---Package[tuple]{Unknown} - --1008013035164995818
+| |---Package[tuple]{Unknown} - -2312201225366906989
| Union[tuple] - -38
| |
-| |---Local Rearrange[tuple]{Unknown}(false) - -3900012572437255236
+| |---Local Rearrange[tuple]{Unknown}(false) - --4071762447953696591
| | |
-| | |---Filter[tuple] - -7391599663069134339
+| | |---Filter[tuple] - --6487150145665307346
+| | | |
+| | | Constant(true) - -7123161826157220327
| | |
| | |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -33
| |
-| |---Local Rearrange[tuple]{Unknown}(false) - -8760996681222683693
+| |---Local Rearrange[tuple]{Unknown}(false) - --3509976263222494134
| |
-| |---Filter[tuple] - -956528893337238225
+| |---Filter[tuple] - -340188275237520979
+| | |
+| | Constant(true) - --5941754500044119766
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -35
|
@@ -20,4 +24,4 @@
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -32
| |
- | |---Load(DummyFil:DummyLdr) - --5840807574562740596
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -2626971498845605352
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld Tue Apr 7 02:29:05 2009
@@ -1,32 +1,40 @@
MapReduce(-1) - -52:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - -8531323295221133204
+| Store(DummyFil:DummyLdr) - -6411189422016119355
| |
-| |---Union[tuple] - -3515731110955624708
+| |---Union[tuple] - -7225341208466719305
| |
-| |---Filter[tuple] - -1351115285619019432
+| |---Filter[tuple] - -551977918718096509
+| | | |
+| | | Constant(true) - --7987250529196129674
| | |
| | |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -48
| |
-| |---Filter[tuple] - -6869916393445222618
+| |---Filter[tuple] - -2056844330428347996
+| | |
+| | Constant(true) - --8754957318949788629
| |
| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -50
|
|---MapReduce(-1) - -45:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -47
| |
- | |---Package[tuple]{Unknown} - --1596982214453270401
+ | |---Package[tuple]{Unknown} - --1607475648664293401
| Union[tuple] - -46
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - --6728652914243238289
+ | |---Local Rearrange[tuple]{Unknown}(false) - --3476413267732334825
| | |
- | | |---Filter[tuple] - --3509976263222494134
+ | | |---Filter[tuple] - -1351115285619019432
+ | | | |
+ | | | Constant(true) - -3515731110955624708
| | |
| | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -41
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - --4071762447953696591
+ | |---Local Rearrange[tuple]{Unknown}(false) - -2351257193210757136
| |
- | |---Filter[tuple] - -7123161826157220327
+ | |---Filter[tuple] - -8531323295221133204
+ | | |
+ | | Constant(true) - -728209667098611077
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -43
|
@@ -34,4 +42,4 @@
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -40
| |
- | |---Load(DummyFil:DummyLdr) - --5941754500044119766
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --7797378030368227515
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld Tue Apr 7 02:29:05 2009
@@ -1,46 +1,58 @@
MapReduce(100) - -74:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - -4747874603499386537
+| Store(DummyFil:DummyLdr) - -7034055442337723447
| |
-| |---Union[tuple] - --7231511798448197372
+| |---Union[tuple] - -2910267342587130351
| |
-| |---Filter[tuple] - --7091196022628815072
+| |---Filter[tuple] - --2136667002319908593
+| | | |
+| | | Constant(true) - --6380139708906526553
| | |
| | |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -70
| |
-| |---Filter[tuple] - -5984722006991913573
+| |---Filter[tuple] - -2468262982799101733
+| | |
+| | Constant(true) - -7123718043666602037
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -72
|
|---MapReduce(40) - -67:
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -69
| |
- | |---Package[tuple]{Unknown} - -8067897495211048679
+ | |---Package[tuple]{Unknown} - --7096529877081178520
| Union[tuple] - -68
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - --8754957318949788629
+ | |---Local Rearrange[tuple]{Unknown}(false) - --7868505214447593853
| | |
- | | |---Filter[tuple] - -551977918718096509
+ | | |---Filter[tuple] - -4721502244557927278
+ | | | |
+ | | | Constant(true) - -8729990799028586395
| | |
- | | |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -58
+ | | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -58
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - -6411189422016119355
+ | |---Local Rearrange[tuple]{Unknown}(false) - --5177745552827005198
| | |
- | | |---Filter[tuple] - -2056844330428347996
+ | | |---Filter[tuple] - --7681398237172009051
+ | | | |
+ | | | Constant(true) - --5723332042410534632
| | |
- | | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -63
+ | | |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -63
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - -7225341208466719305
+ | |---Local Rearrange[tuple]{Unknown}(false) - -1236980712440527800
| |
- | |---Filter[tuple] - --7987250529196129674
+ | |---Filter[tuple] - -6790164331631653200
+ | | |
+ | | Constant(true) - --6430355428631435461
| |
| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -65
|
- |---MapReduce(10) - -56:
+ |---MapReduce(20) - -56:
| Reduce Plan Empty
- | | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -57
+ | | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -57
| | |
- | | |---Filter[tuple] - -2351257193210757136
+ | | |---Filter[tuple] - -5984722006991913573
+ | | | |
+ | | | Constant(true) - --7231511798448197372
| | |
| | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -55
| |
@@ -48,13 +60,15 @@
| Reduce Plan Empty
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
| | |
- | | |---Load(DummyFil:DummyLdr) - -728209667098611077
+ | | |---Load(DummyFil:DummyLdr) - -8816201798088731301
|
- |---MapReduce(20) - -61:
+ |---MapReduce(10) - -61:
Reduce Plan Empty
- | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -62
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -62
| |
- | |---Filter[tuple] - --1369132270304461038
+ | |---Filter[tuple] - -8205341881606740757
+ | | |
+ | | Constant(true) - --7091196022628815072
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -60
|
@@ -62,4 +76,4 @@
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
| |
- | |---Load(DummyFil:DummyLdr) - -728209667098611077
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -8816201798088731301
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Tue Apr 7 02:29:05 2009
@@ -1,43 +1,43 @@
MapReduce(1,GFCross) - -153:
-| Store(DummyFil:DummyLdr) - --5683415113785058706
+| Store(DummyFil:DummyLdr) - --6405280822876279137
| |
-| |---New For Each(false)[tuple] - --8002381389674382470
+| |---New For Each(false)[tuple] - -6741648588878535427
| | |
-| | POUserFunc(org.apache.pig.impl.builtin.GFCross)[tuple] - -727770031531364881
+| | POUserFunc(org.apache.pig.impl.builtin.GFCross)[tuple] - -6101282397594931701
| | |
-| | |---Project[tuple][*] - -7128285064986147947
+| | |---Project[tuple][*] - -5692939642430999118
| |
-| |---Package[tuple]{Unknown} - --885269774183211482
-| Local Rearrange[tuple]{Unknown}(false) - --776319888013965510
+| |---Package[tuple]{Unknown} - -6079615556647418436
+| Local Rearrange[tuple]{Unknown}(false) - -8219725798912083822
| |
| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -152
|
|---MapReduce(1,AVG) - -150:
| Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -151
| |
- | |---New For Each(false)[tuple] - -7965768498188214494
+ | |---New For Each(false)[tuple] - --2783416442434419494
| | |
- | | POUserFunc(org.apache.pig.builtin.AVG)[tuple] - --4867358672373443663
+ | | POUserFunc(org.apache.pig.builtin.AVG)[tuple] - --3889827013424534115
| | |
- | | |---Project[tuple][*] - -31712229583931650
+ | | |---Project[tuple][*] - -7714111612268358662
| |
- | |---Package[tuple]{Unknown} - --7335024873119453444
- | Local Rearrange[tuple]{Unknown}(false) - -4589138876054328603
+ | |---Package[tuple]{Unknown} - --1613182091613226659
+ | Local Rearrange[tuple]{Unknown}(false) - -5165956429696944631
| |
| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -149
|
|---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -142:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -148
| |
- | |---New For Each(false,false)[tuple] - --4248200967728536480
+ | |---New For Each(false,false)[tuple] - -2197807331204639125
| | |
- | | POUserFunc(org.apache.pig.builtin.COUNT)[tuple] - -8767305735755351861
+ | | POUserFunc(org.apache.pig.builtin.COUNT)[tuple] - -5878058249424113280
| | |
- | | |---Project[tuple][*] - --5908426805312852480
+ | | |---Project[tuple][*] - -7085374960058856883
| | |
- | | POUserFunc(org.apache.pig.builtin.SUM)[tuple] - -4744867519639813717
+ | | POUserFunc(org.apache.pig.builtin.SUM)[tuple] - --8447156979210379761
| | |
- | | |---Project[tuple][*] - --1848504978980807369
+ | | |---Project[tuple][*] - -6139496040975471496
| |
| |---New For Each(true)[tuple] - -147
| | |
@@ -57,7 +57,7 @@
| | |
| | Constant(20) - -138
| | |
- | | POSort[bag](org.apache.pig.test.TestMRCompiler$WeirdComparator) - --8479692259657755370
+ | | POSort[bag](org.apache.pig.test.TestMRCompiler$WeirdComparator) - -4188863770717253580
| | | |
| | | Project[tuple][*] - -137
| | |
@@ -78,6 +78,8 @@
Reduce Plan Empty
| Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -128
| |
- | |---Filter[tuple] - -9193928674704944093
+ | |---Filter[tuple] - -2082992246427879202
+ | | |
+ | | Constant(true) - --6860857982727545176
| |
- | |---Load(DummyFil:DummyLdr) - --4238531569995320849
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld Tue Apr 7 02:29:05 2009
@@ -1,5 +1,5 @@
MapReduce(-1) - -167:
-| Store(DummyFil:DummyLdr) - -7973970339130605847
+| Store(DummyFil:DummyLdr) - --696216369324861973
| |
| |---New For Each(true)[bag] - -170
| | |
@@ -15,8 +15,8 @@
|---MapReduce(-1) - -162:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -165
| |
- | |---Package[tuple]{Unknown} - -2082992246427879202
- | Local Rearrange[tuple]{Unknown}(false) - --3148893660811981376
+ | |---Package[tuple]{Unknown} - -2975419344702132532
+ | Local Rearrange[tuple]{Unknown}(false) - -6555138338004402415
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -161
|
@@ -32,6 +32,8 @@
| | |
| | Project[tuple][*] - -155
| |
- | |---Filter[tuple] - --7926255547935388282
+ | |---Filter[tuple] - --8322891634142946616
+ | | |
+ | | Constant(true) - -7251706569645724914
| |
- | |---Load(DummyFil:DummyLdr) - -8639002377389679348
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --7448840386612739853
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld Tue Apr 7 02:29:05 2009
@@ -1,5 +1,5 @@
MapReduce(1) - -171:
-| Store(DummyFil:DummyLdr) - -7856319821130535798
+| Store(DummyFil:DummyLdr) - -3851605818031718348
| |
| |---Limit - -177
| |
@@ -12,6 +12,6 @@
| | |
| | Project[tuple][*] - -172
| |
-| |---Limit - -7398260302074824818
+| |---Limit - --8049873144002881309
| |
-| |---Load(DummyFil:DummyLdr) - -4188863770717253580
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - --3658165997714269604
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld Tue Apr 7 02:29:05 2009
@@ -1,8 +1,8 @@
MapReduce(-1) - -81:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - -7034055442337723447
+| Store(DummyFil:DummyLdr) - --2626287810923037076
| |
-| |---Union[tuple] - -2910267342587130351
+| |---Union[tuple] - --5768272326302808468
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -82
| |
@@ -11,15 +11,15 @@
|---MapReduce(-1) - -79:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -83
| | |
-| | |---Package[tuple]{Unknown} - -370083002387034672
-| | Local Rearrange[tuple]{Unknown}(false) - -6305091296204163466
+| | |---Package[tuple]{Unknown} - --2829086180578157275
+| | Local Rearrange[tuple]{Unknown}(false) - -6435233330554227045
| | |
-| | |---Load(DummyFil:DummyLdr) - -1236980712440527800
+| | |---Load(DummyFil:DummyLdr) - --4168060277593001906
|
|---MapReduce(-1) - -80:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -85
| |
- | |---Package[tuple]{Unknown} - -7123718043666602037
- | Local Rearrange[tuple]{Unknown}(false) - --6380139708906526553
+ | |---Package[tuple]{Unknown} - -7483213803049293823
+ | Local Rearrange[tuple]{Unknown}(false) - --5190071275724378681
| |
- | |---Load(DummyFil:DummyLdr) - --2136667002319908593
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --6957591191795645940
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld Tue Apr 7 02:29:05 2009
@@ -1,29 +1,31 @@
MapReduce(-1) - -89:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - --5190071275724378681
+| Store(DummyFil:DummyLdr) - -3719468117239527682
| |
-| |---Union[tuple] - --3396897091865664764
+| |---Union[tuple] - --2530083186736419352
| |
-| |---Filter[tuple] - --6957591191795645940
+| |---Filter[tuple] - --3624738819541751735
+| | | |
+| | | Constant(true) - -1761200043915140459
| | |
-| | |---Load(DummyFil:DummyLdr) - -5515425171581967372
+| | |---Load(DummyFil:DummyLdr) - --4776862487779398064
| |
| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -90
| |
| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -92
|
-|---MapReduce(-1) - -87:
+|---MapReduce(-1) - -86:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -91
| | |
-| | |---Package[tuple]{Unknown} - --6061281703859425960
-| | Local Rearrange[tuple]{Unknown}(false) - --1158897849427419546
+| | |---Package[tuple]{Unknown} - --3671186819751472084
+| | Local Rearrange[tuple]{Unknown}(false) - -3737603423295312892
| | |
-| | |---Load(DummyFil:DummyLdr) - -3709512757404691843
+| | |---Load(DummyFil:DummyLdr) - --5123390619301085966
|
|---MapReduce(-1) - -88:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -93
| |
- | |---Package[tuple]{Unknown} - --2057425961601007773
- | Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
+ | |---Package[tuple]{Unknown} - --2325244147060806375
+ | Local Rearrange[tuple]{Unknown}(false) - --1194577301115518934
| |
- | |---Load(DummyFil:DummyLdr) - -7506868571066332964
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --8027742474430787324
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC5.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC5.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC5.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC5.gld Tue Apr 7 02:29:05 2009
@@ -1,13 +1,17 @@
MapReduce(-1) - -96:
Reduce Plan Empty
-| Store(DummyFil:DummyLdr) - --2829086180578157275
+| Store(DummyFil:DummyLdr) - -7127277012934370361
| |
-| |---Union[tuple] - --5371705948522929752
+| |---Union[tuple] - --1036765666284482668
| |
-| |---Filter[tuple] - -6435233330554227045
+| |---Filter[tuple] - --5111685507913827932
+| | | |
+| | | Constant(true) - -7860460526876129822
| | |
-| | |---Load(DummyFil:DummyLdr) - --4168060277593001906
+| | |---Load(DummyFil:DummyLdr) - --2426993543147308005
| |
-| |---Filter[tuple] - -7483213803049293823
+| |---Filter[tuple] - -3683227376238667289
+| | |
+| | Constant(true) - --8622295867288126988
| |
-| |---Load(DummyFil:DummyLdr) - -580704905698185869
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - -810744320494301308
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld?rev=762596&r1=762595&r2=762596&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld Tue Apr 7 02:29:05 2009
@@ -1,7 +1,7 @@
MapReduce(-1) - -99:
-| Store(DummyFil:DummyLdr) - --3624738819541751735
+| Store(DummyFil:DummyLdr) - -8554221887998033529
| |
-| |---Package[tuple]{Unknown} - --4776862487779398064
+| |---Package[tuple]{Unknown} - -5969405952749301492
| Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -100
|
| Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -102
@@ -9,19 +9,19 @@
|---MapReduce(-1) - -97:
| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -101
| | |
-| | |---Local Rearrange[tuple]{Unknown}(false) - --3671186819751472084
+| | |---Local Rearrange[tuple]{Unknown}(false) - --203604194309414000
| | |
-| | |---Package[tuple]{Unknown} - -3737603423295312892
-| | Local Rearrange[tuple]{Unknown}(false) - --2325244147060806375
+| | |---Package[tuple]{Unknown} - --2216790928607759809
+| | Local Rearrange[tuple]{Unknown}(false) - -7219092146411102494
| | |
-| | |---Load(DummyFil:DummyLdr) - --8240903279973257769
+| | |---Load(DummyFil:DummyLdr) - -4266652190633805129
|
|---MapReduce(-1) - -98:
| Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -103
| |
- | |---Local Rearrange[tuple]{Unknown}(false) - --3008031181120208412
+ | |---Local Rearrange[tuple]{Unknown}(false) - -8515950928528386562
| |
- | |---Package[tuple]{Unknown} - --1194577301115518934
- | Local Rearrange[tuple]{Unknown}(false) - --2626287810923037076
+ | |---Package[tuple]{Unknown} - -8957152010410190649
+ | Local Rearrange[tuple]{Unknown}(false) - --2465563180022385815
| |
- | |---Load(DummyFil:DummyLdr) - --5768272326302808468
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - --506205335735031954
\ No newline at end of file