You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/11/02 07:18:53 UTC
svn commit: r1196466 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/newplan/logical/optimizer/ test/org/apache/pig/test/
Author: daijy
Date: Wed Nov 2 06:18:53 2011
New Revision: 1196466
URL: http://svn.apache.org/viewvc?rev=1196466&view=rev
Log:
PIG-2119: DuplicateForEachColumnRewrite makes assumptions about the position of LOGGenerate in the plan
Added:
pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Nov 2 06:18:53 2011
@@ -156,6 +156,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2119: DuplicateForEachColumnRewrite makes assumptions about the position of LOGGenerate in the plan (daijy)
+
PIG-2290: TOBAG wraps tuple parameters in another tuple (ryan.hoegg via thejas)
PIG-2288: Pig 0.9 error message not useful as compared to 0.8 in case
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Wed Nov 2 06:18:53 2011
@@ -48,6 +48,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.optimizer.UidResetter;
@@ -236,6 +237,9 @@ public class HExecutionEngine {
pod.visit();
}
+ DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover( plan );
+ DanglingNestedNodeRemover.visit();
+
UidResetter uidResetter = new UidResetter( plan );
uidResetter.visit();
Added: pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java?rev=1196466&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java (added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java Wed Nov 2 06:18:53 2011
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.newplan.logical.optimizer;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+
+public class DanglingNestedNodeRemover extends LogicalRelationalNodesVisitor {
+
+ public DanglingNestedNodeRemover(OperatorPlan plan)
+ throws FrontendException {
+ super(plan, new DependencyOrderWalker(plan));
+ }
+
+ @Override
+ public void visit(LOForEach foreach) throws FrontendException {
+ LogicalPlan innerPlan = foreach.getInnerPlan();
+ List<Operator> opsToRemove = new ArrayList<Operator>();
+ Iterator<Operator> ops = innerPlan.getOperators();
+
+ while (ops.hasNext()) {
+ Operator op = ops.next();
+ // Check if op leads to LOGenerate, otherwise, candidate to remove
+ Operator currentOp = op;
+ boolean endWithNoLOGenerate = false;
+ while (!(currentOp instanceof LOGenerate)) {
+ if (innerPlan.getSuccessors(currentOp)==null) {
+ endWithNoLOGenerate = true;
+ break;
+ }
+ currentOp = innerPlan.getSuccessors(currentOp).get(0);
+ }
+ if (endWithNoLOGenerate)
+ opsToRemove.add(op);
+ }
+
+ for (Operator op : opsToRemove) {
+ innerPlan.removeAndReconnect(op);
+ }
+ }
+}
Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Wed Nov 2 06:18:53 2011
@@ -175,4 +175,20 @@ public class TestPlanGeneration extends
poStore = (POStore)mrOper.mapPlan.getLeaves().get(0);
assert(poStore.getAlias().equals("B"));
}
+
+ // See PIG-2119
+ @Test
+ public void testDanglingNestedNode() throws Exception {
+ String query = "a = load 'b.txt' AS (id:chararray, num:int); " +
+ "b = group a by id;" +
+ "c = foreach b {" +
+ " d = order a by num DESC;" +
+ " n = COUNT(a);" +
+ " e = limit d 1;" +
+ " generate n;" +
+ "};";
+
+ LogicalPlan lp = Util.parse(query, pc);
+ Util.optimizeNewLP(lp);
+ }
}
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Wed Nov 2 06:18:53 2011
@@ -82,6 +82,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.relational.LOStore;
@@ -721,6 +722,9 @@ public class Util {
public static LogicalPlan optimizeNewLP(
LogicalPlan lp)
throws FrontendException{
+ DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover( lp );
+ DanglingNestedNodeRemover.visit();
+
UidResetter uidResetter = new UidResetter( lp );
uidResetter.visit();