You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2014/01/30 18:00:55 UTC
svn commit: r1562894 - in /pig/trunk: ./
src/org/apache/pig/newplan/logical/relational/
src/org/apache/pig/newplan/logical/visitor/ src/org/apache/pig/parser/
test/e2e/pig/tests/
Author: knoguchi
Date: Thu Jan 30 17:00:55 2014
New Revision: 1562894
URL: http://svn.apache.org/r1562894
Log:
PIG-3641: Split "otherwise" producing incorrect output when combined with ColumnPruning (knoguchi)
Added:
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ResetProjectionAttachedRelationalOpVisitor.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1562894&r1=1562893&r2=1562894&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jan 30 17:00:55 2014
@@ -76,6 +76,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3641: Split "otherwise" producing incorrect output when combined with ColumnPruning (knoguchi)
+
PIG-3682: mvn-inst target does not install pig-h2.jar into local .m2 (raluri via aniket486)
PIG-3661: Piggybank AvroStorage fails if used in more than one load or store statement (rohini)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java?rev=1562894&r1=1562893&r2=1562894&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java Thu Jan 30 17:00:55 2014
@@ -31,8 +31,8 @@ import org.apache.pig.newplan.OperatorPl
import org.apache.pig.newplan.PlanVisitor;
import org.apache.pig.newplan.ReverseDependencyOrderWalker;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
-import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.visitor.ResetProjectionAttachedRelationalOpVisitor;
public class LOSort extends LogicalRelationalOperator{
private List<Boolean> mAscCols;
@@ -200,23 +200,6 @@ public class LOSort extends LogicalRelat
return plan.getPredecessors(this).get(0);
}
- private static class ResetProjectionAttachedRelationalOpVisitor
- extends LogicalExpressionVisitor {
- private LogicalRelationalOperator attachedRelationalOp;
-
- ResetProjectionAttachedRelationalOpVisitor (
- LogicalExpressionPlan plan, LogicalRelationalOperator op )
- throws FrontendException {
- super(plan, new ReverseDependencyOrderWalker(plan));
- this.attachedRelationalOp = op;
-
- }
- @Override
- public void visit(ProjectExpression pe) throws FrontendException {
- pe.setAttachedRelationalOp(attachedRelationalOp);
- }
- }
-
public static LOSort createCopy(LOSort sort) throws FrontendException {
LOSort newSort = new LOSort(sort.getPlan(), null,
sort.getAscendingCols(),
Added: pig/trunk/src/org/apache/pig/newplan/logical/visitor/ResetProjectionAttachedRelationalOpVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ResetProjectionAttachedRelationalOpVisitor.java?rev=1562894&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/ResetProjectionAttachedRelationalOpVisitor.java (added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/ResetProjectionAttachedRelationalOpVisitor.java Thu Jan 30 17:00:55 2014
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.newplan.logical.visitor;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.ReverseDependencyOrderWalker;
+import org.apache.pig.newplan.logical.expression.LogicalExpression;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
+import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
+
+public class ResetProjectionAttachedRelationalOpVisitor
+ extends LogicalExpressionVisitor {
+ private LogicalRelationalOperator attachedRelationalOp;
+
+ public ResetProjectionAttachedRelationalOpVisitor (
+ LogicalExpressionPlan plan, LogicalRelationalOperator op )
+ throws FrontendException {
+ super(plan, new ReverseDependencyOrderWalker(plan));
+ this.attachedRelationalOp = op;
+
+ }
+
+ @Override
+ public void visit(ProjectExpression pe) throws FrontendException {
+ pe.setAttachedRelationalOp(attachedRelationalOp);
+ }
+}
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1562894&r1=1562893&r2=1562894&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Thu Jan 30 17:00:55 2014
@@ -99,6 +99,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.rules.OptimizerUtils;
import org.apache.pig.newplan.logical.visitor.ProjStarInUdfExpander;
import org.apache.pig.newplan.logical.visitor.ProjectStarExpander;
+import org.apache.pig.newplan.logical.visitor.ResetProjectionAttachedRelationalOpVisitor;
public class LogicalPlanBuilder {
@@ -296,6 +297,18 @@ public class LogicalPlanBuilder {
}
// using De Morgan's law (!A && !B) == !(A || B)
currentExpr = new NotExpression(splitPlan, currentExpr);
+
+ try {
+ // Going through all the ProjectExpressions that were cloned
+ // and updating the attached operators from its original
+ // LOSplitOutput to to the "otherwise" LOSplitOutput
+ // (PIG-3641)
+ new ResetProjectionAttachedRelationalOpVisitor(splitPlan, op).visit();
+ } catch (FrontendException e) {
+ e.printStackTrace();
+ throw new PlanGenerationFailureException(intStream, loc, e);
+ }
+
op.setFilterPlan(splitPlan);
return buildOp(loc, op, alias, inputAlias, null);
}
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1562894&r1=1562893&r2=1562894&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Thu Jan 30 17:00:55 2014
@@ -4793,6 +4793,24 @@ store C into ':OUTPATH:';\,
I = limit H 3;
J = foreach I generate contributions;
STORE J INTO ':OUTPATH:.2';?,
+ }, {
+ # PIG-3641
+ 'num' => 6,
+ 'pig' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions);
+ -- dropping one column to force columnprune
+ B = foreach A generate name, age, registration;
+ -- Next line is the only difference
+ SPLIT B into C1 if age > 50, C2 otherwise;
+ D1 = foreach C1 generate age, registration;
+ STORE D1 INTO ':OUTPATH:.1';
+ STORE C2 INTO ':OUTPATH:.2';?,
+ 'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions);
+ -- dropping one column to force columnprune
+ B = foreach A generate name, age, registration;
+ SPLIT B into C1 if age > 50, C2 if age <= 50;
+ D1 = foreach C1 generate age, registration;
+ STORE D1 INTO ':OUTPATH:.1';
+ STORE C2 INTO ':OUTPATH:.2';?,
}
],
},{