svn commit: r693574 - in /incubator/pig/branches/types/src/org/apache/pig: PigServer.java backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java impl/logicalLayer/optimizer/LogicalTransformer.java

[ga...@apache.org]

Author: gates
Date: Tue Sep  9 12:32:38 2008
New Revision: 693574

URL: http://svn.apache.org/viewvc?rev=693574&view=rev
Log:
PIG-416 Fixed LogicalTransformer so that when a script with a split needs to be reorganized after an optimization, the LOSplitOutput is properly patched up.
Previously it was getting skipped.

Fixed PigServer so that explain will work after a split.

Removed debug statement from PigMapReduce that was printing every line out as it went into reduce.


Modified:
    incubator/pig/branches/types/src/org/apache/pig/PigServer.java
    incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java

Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/PigServer.java?rev=693574&r1=693573&r2=693574&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/PigServer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/PigServer.java Tue Sep  9 12:32:38 2008
@@ -398,7 +398,11 @@
             if(null == op) {
                 throw new IOException("Unable to find an operator for alias " + alias);
             }
-            LogicalPlan lp = compileLp(getPlanFromAlias(alias, op.getClass().getName()), "explain");
+            LogicalPlan storePlan = QueryParser.generateStorePlan(opTable,
+                scope, getPlanFromAlias(alias, op.getClass().getName()),
+                "fakefile", PigStorage.class.getName(), aliasOp.get(alias),
+                aliases);
+            LogicalPlan lp = compileLp(storePlan, "explain");
             stream.println("Logical Plan:");
             LOPrinter lv = new LOPrinter(stream, lp);
             lv.visit();

Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java?rev=693574&r1=693573&r2=693574&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java Tue Sep  9 12:32:38 2008
@@ -167,7 +167,6 @@
                         return;
                     }
                     
-                    log.info("Attaching " + packRes + " to " + rp.getRoots());
                     rp.attachInput(packRes);
 
                     List<PhysicalOperator> leaves = rp.getLeaves();

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java?rev=693574&r1=693573&r2=693574&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java Tue Sep  9 12:32:38 2008
@@ -21,6 +21,9 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import org.apache.pig.impl.plan.DepthFirstWalker;
 import org.apache.pig.impl.plan.PlanWalker;
 import org.apache.pig.impl.plan.VisitorException;
@@ -41,6 +44,8 @@
 
 public abstract class LogicalTransformer extends Transformer<LogicalOperator, LogicalPlan> {
 
+    private final Log log = LogFactory.getLog(getClass());
+
     protected LogicalTransformer(
             LogicalPlan plan,
             PlanWalker<LogicalOperator, LogicalPlan> walker) {
@@ -185,8 +190,19 @@
             plans.addAll(((LOSort)before).getSortColPlans());
         } else if (before instanceof LOFilter) {
             plans.add(((LOFilter)before).getComparisonPlan());
-        } else if (before instanceof LOSplitOutput) {
-            plans.add(((LOSplitOutput)before).getConditionPlan());
+        } else if (before instanceof LOSplit) {
+            // In this case we have to find each of the Split outputs, and
+            // add their plans.
+            List<LogicalOperator> splitOutputs = mPlan.getSuccessors(before);
+            for (LogicalOperator so : splitOutputs) {
+                if (!(so instanceof LOSplitOutput)) {
+                    String msg = "Found an LOSplit with an operator other " 
+                        + "than LOSplitOutput after it!";
+                    log.error(msg);
+                    throw new VisitorException(msg);
+                }
+                plans.add(((LOSplitOutput)so).getConditionPlan());
+            }
         } else if (before instanceof LOForEach) {
             plans.addAll(((LOForEach)before).getForEachPlans());
         }