You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/03/01 05:13:46 UTC
svn commit: r632546 - in /incubator/pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/local/executionengine/
src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/physicalL...
Author: gates
Date: Fri Feb 29 20:13:45 2008
New Revision: 632546
URL: http://svn.apache.org/viewvc?rev=632546&view=rev
Log:
PIG-113 Make explain output more understandable.
Added:
incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java
incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java
incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java
incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=632546&r1=632545&r2=632546&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Fri Feb 29 20:13:45 2008
@@ -147,3 +147,5 @@
gates).
PIG-13: adding version to the system (joa23 via olgan)
+
+ PIG-113: Make explain output more understandable (pi_song via gates)
Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java Fri Feb 29 20:13:45 2008
@@ -10,6 +10,7 @@
import org.apache.pig.backend.executionengine.ExecPhysicalPlan;
import org.apache.pig.impl.physicalLayer.PhysicalOperator;
import org.apache.pig.impl.physicalLayer.POPrinter;
+import org.apache.pig.impl.physicalLayer.POTreePrinter;
import org.apache.pig.impl.physicalLayer.POVisitor;
import org.apache.pig.impl.logicalLayer.OperatorKey;
@@ -36,8 +37,7 @@
}
public void explain(OutputStream out) {
- POVisitor lprinter = new POPrinter(opTable, new PrintStream(out));
-
+ POVisitor lprinter = new POTreePrinter(opTable, new PrintStream(out));
((PhysicalOperator)opTable.get(root)).visit(lprinter);
}
Modified: incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java Fri Feb 29 20:13:45 2008
@@ -8,6 +8,7 @@
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.executionengine.ExecPhysicalOperator;
import org.apache.pig.backend.executionengine.ExecPhysicalPlan;
+import org.apache.pig.impl.physicalLayer.POTreePrinter;
import org.apache.pig.impl.physicalLayer.PhysicalOperator;
import org.apache.pig.impl.physicalLayer.POVisitor;
import org.apache.pig.impl.physicalLayer.POPrinter;
@@ -35,8 +36,7 @@
}
public void explain(OutputStream out) {
- POVisitor lprinter = new POPrinter(opTable, new PrintStream(out));
-
+ POVisitor lprinter = new POTreePrinter(opTable, new PrintStream(out));
((PhysicalOperator)opTable.get(root)).visit(lprinter);
}
Added: incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java?rev=632546&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java Fri Feb 29 20:13:45 2008
@@ -0,0 +1,124 @@
+package org.apache.pig.impl.eval;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+
+import org.apache.pig.impl.eval.cond.AndCond;
+import org.apache.pig.impl.eval.cond.CompCond;
+import org.apache.pig.impl.eval.cond.FalseCond;
+import org.apache.pig.impl.eval.cond.FuncCond;
+import org.apache.pig.impl.eval.cond.NotCond;
+import org.apache.pig.impl.eval.cond.OrCond;
+import org.apache.pig.impl.eval.cond.RegexpCond;
+import org.apache.pig.impl.eval.cond.TrueCond;
+
+public class EvalSpecTreePrinter extends EvalSpecVisitor {
+ private PrintStream mStream = null;
+
+ public EvalSpecTreePrinter(PrintStream ps) {
+ mStream = ps;
+ }
+
+ @Override
+ public void visitFilter(FilterSpec f) {
+ mStream.print("Filter: ");
+ if (f.cond instanceof AndCond) mStream.print(" AND ");
+ else if (f.cond instanceof CompCond) mStream.print(" COMP ");
+ else if (f.cond instanceof FalseCond) mStream.print(" FALSE ");
+ else if (f.cond instanceof FuncCond) mStream.print(" FUNC ");
+ else if (f.cond instanceof NotCond) mStream.print(" NOT ");
+ else if (f.cond instanceof OrCond) mStream.print(" OR ");
+ else if (f.cond instanceof RegexpCond) mStream.print(" REGEXP ");
+ else if (f.cond instanceof TrueCond) mStream.print(" TRUE ");
+ else throw new AssertionError(" Unknown Cond ");
+ }
+
+ @Override
+ public void visitSortDistinct(SortDistinctSpec sd) {
+ mStream.print("Sort(") ;
+ if (sd.distinct()) {
+ mStream.print("Distinct(") ;
+ }
+ sd.getSortSpec().visit(this);
+ if (sd.distinct()) {
+ mStream.print(")") ;
+ }
+ mStream.print(")") ;
+ }
+
+ @Override
+ public void visitGenerate(GenerateSpec g) {
+ mStream.print("Generate(");
+ Iterator<EvalSpec> i = g.getSpecs().iterator();
+ boolean isFirst = true ;
+ while (i.hasNext()) {
+ if (isFirst) {
+ isFirst = false ;
+ }
+ else {
+ mStream.print(",") ;
+ }
+ i.next().visit(this);
+ }
+ mStream.print(")") ;
+ }
+
+ @Override
+ public void visitMapLookup(MapLookupSpec ml) {
+ mStream.print("MapLookup(key=" + ml.key() + ")");
+ }
+
+ @Override
+ public void visitConst(ConstSpec c) {
+ mStream.print("Const(" + c.value() + ")");
+ }
+
+ @Override
+ public void visitProject(ProjectSpec p) {
+ mStream.print("Project(");
+ Iterator<Integer> i = p.getCols().iterator();
+ while (i.hasNext()) {
+ mStream.print(i.next().intValue());
+ if (i.hasNext()) mStream.print(",");
+ }
+ mStream.print(")");
+ }
+
+ @Override
+ public void visitStar(StarSpec s) {
+ mStream.print("*");
+ }
+
+ @Override
+ public void visitFuncEval(FuncEvalSpec fe) {
+ mStream.print("FuncEval(" + fe.getFuncName() + "(");
+ fe.getArgs().visit(this);
+ mStream.print("))") ;
+ }
+
+ @Override
+ public void visitCompositeEval(CompositeEvalSpec ce) {
+ mStream.print("Composite(");
+ Iterator<EvalSpec> i = ce.getSpecs().iterator();
+ boolean isFirst = true ;
+ while (i.hasNext()) {
+ if (isFirst) {
+ isFirst = false ;
+ }
+ else {
+ mStream.print(",") ;
+ }
+ i.next().visit(this);
+ }
+ mStream.print(")") ;
+ }
+
+ @Override
+ public void visitBinCond(BinCondSpec bc) {
+ mStream.print("BinCond(True=>");
+ bc.ifTrue().visit(this);
+ mStream.print (",False=>");
+ bc.ifFalse().visit(this);
+ mStream.print (")");
+ }
+}
Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java?rev=632546&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java Fri Feb 29 20:13:45 2008
@@ -0,0 +1,151 @@
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.eval.cond.Cond;
+
+import org.apache.pig.impl.eval.EvalSpec;
+import org.apache.pig.impl.eval.EvalSpecPrinter;
+
+public class LOTreePrinter extends LOVisitor {
+
+ private static final String LINE_START_SYMBOL = "|---" ;
+ private static final String IDENT_SYMBOL = " " ;
+
+ private int currentIdent = 0 ;
+
+ private PrintStream mStream = null;
+
+ public LOTreePrinter(PrintStream ps) {
+ mStream = ps;
+ }
+
+ public void increaseIdent() {
+ currentIdent++ ;
+ }
+
+ public void decreaseIdent() {
+ currentIdent-- ;
+ if (currentIdent <0)
+ {
+ throw new RuntimeException("Invalid LOTreePrinter state. currentIdent < 0") ;
+ }
+ }
+
+ @Override
+ public void visitCogroup(LOCogroup g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( " + g.arguments() + " ) ") ;
+ mStream.println() ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitEval(LOEval g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( " + g.arguments() + " ) ") ;
+ mStream.println() ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitUnion(LOUnion g) {
+ printLineHeader(g) ;
+ depthFirstSearchVisit(g) ;
+ }
+
+
+ @Override
+ public void visitLoad(LOLoad g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( file = " + g.getInputFileSpec().getFileName() ) ;
+
+ if (g.outputSchema().getFields().size() > 0)
+ {
+ mStream.print(" AS ") ;
+ boolean isFirst = true ;
+ for(Schema schema: g.outputSchema().getFields() ) {
+ if (isFirst) {
+ isFirst = false ;
+ }
+ else {
+ mStream.print(",") ;
+ }
+ mStream.print(schema.getAlias()) ;
+ }
+ }
+
+ mStream.println(" )") ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitSort(LOSort g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( BY " + g.arguments() + " ) ") ;
+ mStream.println() ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitSplit(LOSplit g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( ") ;
+ boolean isFirst = true ;
+ for(Cond cond:g.getConditions()) {
+ if (isFirst) {
+ isFirst = false ;
+ }
+ else {
+ mStream.print(",") ;
+ }
+ mStream.print(cond) ;
+ }
+ mStream.println(" ) ") ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitSplitOutput(LOSplitOutput g) {
+ printLineHeader(g, false) ;
+ mStream.print(" ( " + g.arguments() + " ) ") ;
+ mStream.println() ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ @Override
+ public void visitStore(LOStore g) {
+ printLineHeader(g) ;
+ depthFirstSearchVisit(g) ;
+ }
+
+ private void printLineHeader(LogicalOperator g) {
+ printLineHeader(g, true) ;
+ }
+
+ private void printLineHeader(LogicalOperator g, boolean appendNewLine) {
+ for(int i=0;i<currentIdent;i++) {
+ mStream.print(IDENT_SYMBOL) ;
+ }
+ mStream.print(LINE_START_SYMBOL) ;
+ mStream.print(g.getClass().getSimpleName()) ;
+
+ if (appendNewLine) {
+ mStream.println();
+ }
+ }
+
+ private void depthFirstSearchVisit(LogicalOperator lo) {
+ List<OperatorKey> inputs = lo.getInputs();
+ Iterator<OperatorKey> i = inputs.iterator();
+
+ this.increaseIdent() ;
+ while (i.hasNext()) {
+ LogicalOperator input = lo.getOpTable().get(i.next());
+ input.visit(this);
+ }
+ this.decreaseIdent() ;
+ }
+}
Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Fri Feb 29 20:13:45 2008
@@ -103,7 +103,7 @@
}
public void explain(OutputStream out) {
- LOVisitor lprinter = new LOPrinter(new PrintStream(out));
+ LOVisitor lprinter = new LOTreePrinter(new PrintStream(out));
opTable.get(root).visit(lprinter);
}
Added: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java?rev=632546&view=auto
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java (added)
+++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java Fri Feb 29 20:13:45 2008
@@ -0,0 +1,191 @@
+package org.apache.pig.impl.physicalLayer;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pig.backend.executionengine.ExecPhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.POMapreduce;
+import org.apache.pig.backend.local.executionengine.POCogroup;
+import org.apache.pig.backend.local.executionengine.POEval;
+import org.apache.pig.backend.local.executionengine.POLoad;
+import org.apache.pig.backend.local.executionengine.POSort;
+import org.apache.pig.backend.local.executionengine.POSplit;
+import org.apache.pig.backend.local.executionengine.POStore;
+import org.apache.pig.backend.local.executionengine.POUnion;
+import org.apache.pig.impl.eval.EvalSpec;
+import org.apache.pig.impl.eval.EvalSpecTreePrinter;
+import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
+
+public class POTreePrinter extends POVisitor {
+
+ private static final String HEADER_START_SYMBOL = "|---" ;
+ private static final String CONTENT_START_SYMBOL = " " ;
+ private static final String IDENT_SYMBOL = " " ;
+
+ private int currentIdent = 0 ;
+
+ private PrintStream mStream = null;
+
+ public POTreePrinter(Map<OperatorKey, ExecPhysicalOperator> opTable,
+ PrintStream ps) {
+ super(opTable);
+ mStream = ps;
+ }
+
+ public void increaseIdent() {
+ currentIdent++ ;
+ }
+
+ public void decreaseIdent() {
+ currentIdent-- ;
+ if (currentIdent <0)
+ {
+ throw new RuntimeException("Invalid POTreePrinter state. currentIdent < 0") ;
+ }
+ }
+
+ @Override
+ public void visitMapreduce(POMapreduce p) {
+ printLineHeader(p) ;
+
+ // partition function
+ if (p.partitionFunction != null) {
+ adjustContentIdent() ;
+ mStream.println("Partition Function: " + p.partitionFunction.getName());
+ mStream.println() ;
+ }
+
+ // map line
+ adjustContentIdent() ;
+ mStream.print("Map : ") ;
+ visitSpecs(p.toMap);
+ mStream.println() ;
+
+ // combine line
+ if (p.toCombine != null) {
+ adjustContentIdent() ;
+ mStream.print("Combine : ") ;
+ p.toCombine.visit(new EvalSpecTreePrinter(mStream));
+ mStream.println() ;
+ }
+
+ // reduce line
+ if (p.toReduce != null) {
+ adjustContentIdent() ;
+ mStream.print("Reduce : ") ;
+ p.toReduce.visit(new EvalSpecTreePrinter(mStream));
+ mStream.println() ;
+ }
+
+ // grouping line
+ if (p.groupFuncs != null) {
+ adjustContentIdent() ;
+ mStream.print("Grouping : ") ;
+ visitSpecs(p.groupFuncs);
+ mStream.println() ;
+ }
+
+ // input files line
+ adjustContentIdent() ;
+ mStream.print("Input File(s) : ") ;
+ Iterator<FileSpec> i = p.inputFileSpecs.iterator();
+ while (i.hasNext()) {
+ mStream.print(i.next().getFileName());
+ if (i.hasNext()) mStream.print(", ");
+ }
+ mStream.println() ;
+
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitLoad(POLoad p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitSort(POSort p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitStore(POStore p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitCogroup(POCogroup p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitEval(POEval p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitSplit(POSplit p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ @Override
+ public void visitUnion(POUnion p) {
+ printLineHeader(p) ;
+ depthFirstSearchVisit(p) ;
+ }
+
+ private void printLineHeader(PhysicalOperator g) {
+ printLineHeader(g, true) ;
+ }
+
+ private void printLineHeader(PhysicalOperator g, boolean appendNewLine) {
+ for(int i=0;i<currentIdent;i++) {
+ mStream.print(IDENT_SYMBOL) ;
+ }
+ mStream.print(HEADER_START_SYMBOL) ;
+ mStream.print(g.getClass().getSimpleName()) ;
+
+ if (appendNewLine) {
+ mStream.println();
+ }
+ }
+
+ private void adjustContentIdent() {
+ for(int i=0;i<currentIdent;i++) {
+ mStream.print(IDENT_SYMBOL) ;
+ }
+ mStream.print(CONTENT_START_SYMBOL) ;
+ }
+
+ private void depthFirstSearchVisit(PhysicalOperator po) {
+ this.increaseIdent() ;
+ for(OperatorKey inputKey : po.inputs) {
+ PhysicalOperator input = null ;
+ // since the only sub-type of ExecPhysicalOperator is PhysicalOperator
+ // it is legal to convert this way
+ input = (PhysicalOperator) mOpTable.get(inputKey) ;
+ if (input==null) {
+ throw new RuntimeException("Invalid OpKey table found while reading in POTreePrinter") ;
+ }
+ input.visit(this);
+ }
+ this.decreaseIdent() ;
+ }
+
+ private void visitSpecs(List<EvalSpec> specs) {
+ Iterator<EvalSpec> j = specs.iterator();
+ while (j.hasNext()) {
+ j.next().visit(new EvalSpecTreePrinter(mStream));
+ }
+ }
+
+}