You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2012/12/27 23:11:01 UTC
svn commit: r1426345 - in /pig/branches/branch-0.11: CHANGES.txt
src/org/apache/pig/newplan/logical/relational/LOSort.java
src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
test/e2e/pig/tests/nightly.conf
Author: rohini
Date: Thu Dec 27 22:11:01 2012
New Revision: 1426345
URL: http://svn.apache.org/viewvc?rev=1426345&view=rev
Log:
PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning (knoguchi via rohini)
Modified:
pig/branches/branch-0.11/CHANGES.txt
pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java
pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf
Modified: pig/branches/branch-0.11/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/CHANGES.txt?rev=1426345&r1=1426344&r2=1426345&view=diff
==============================================================================
--- pig/branches/branch-0.11/CHANGES.txt (original)
+++ pig/branches/branch-0.11/CHANGES.txt Thu Dec 27 22:11:01 2012
@@ -332,6 +332,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning (knoguchi via rohini)
+
PIG-3076: make TestScalarAliases more reliable (julien)
PIG-3020: "Duplicate uid in schema" error when joining two relations derived from the same load statement (jcoveney)
Modified: pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java?rev=1426345&r1=1426344&r2=1426345&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java Thu Dec 27 22:11:01 2012
@@ -29,7 +29,9 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.PlanVisitor;
+import org.apache.pig.newplan.ReverseDependencyOrderWalker;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
public class LOSort extends LogicalRelationalOperator{
@@ -197,4 +199,42 @@ public class LOSort extends LogicalRelat
public Operator getInput(LogicalPlan plan) {
return plan.getPredecessors(this).get(0);
}
+
+ private static class ResetProjectionAttachedRelationalOpVisitor
+ extends LogicalExpressionVisitor {
+ private LogicalRelationalOperator attachedRelationalOp;
+
+ ResetProjectionAttachedRelationalOpVisitor (
+ LogicalExpressionPlan plan, LogicalRelationalOperator op )
+ throws FrontendException {
+ super(plan, new ReverseDependencyOrderWalker(plan));
+ this.attachedRelationalOp = op;
+
+ }
+ @Override
+ public void visit(ProjectExpression pe) throws FrontendException {
+ pe.setAttachedRelationalOp(attachedRelationalOp);
+ }
+ }
+
+ public static LOSort createCopy(LOSort sort) throws FrontendException {
+ LOSort newSort = new LOSort(sort.getPlan(), null,
+ sort.getAscendingCols(),
+ sort.getUserFunc());
+
+ List<LogicalExpressionPlan> newSortColPlans =
+ new ArrayList<LogicalExpressionPlan>(sort.getSortColPlans().size());
+
+ for(LogicalExpressionPlan lep:sort.getSortColPlans() ) {
+ LogicalExpressionPlan new_lep = lep.deepCopy();
+
+ // Resetting the attached LOSort operator of the ProjectExpression
+ // to the newSort
+ new ResetProjectionAttachedRelationalOpVisitor(
+ new_lep, newSort ).visit();
+ newSortColPlans.add(new_lep);
+ }
+ newSort.setSortColPlans(newSortColPlans);
+ return newSort;
+ }
}
Modified: pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java?rev=1426345&r1=1426344&r2=1426345&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java Thu Dec 27 22:11:01 2012
@@ -202,9 +202,7 @@ public class LimitOptimizer extends Rule
&& greatGrandparants.size() != 0
&& greatGrandparants.get(0) instanceof LOSort) {
LOSort sort = (LOSort) greatGrandparants.get(0);
- LOSort newSort = new LOSort(sort.getPlan(), sort
- .getSortColPlans(), sort.getAscendingCols(),
- sort.getUserFunc());
+ LOSort newSort = LOSort.createCopy(sort);
newSort.setLimit(limit.getLimit());
currentPlan.replace(limit, newSort);
Modified: pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf?rev=1426345&r1=1426344&r2=1426345&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf (original)
+++ pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf Thu Dec 27 22:11:01 2012
@@ -4575,6 +4575,27 @@ store C into ':OUTPATH:';\,
'rc' => 0,
'not_expected_out_regex' => "checkJobConf: conf is null: false",
'expected_out_regex' => "checkJobConf: conf is null: true",
+ }, {
+ # PIG-3051
+ 'num' => 5,
+ 'pig' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions);
+ -- dropping one column to force columnprune
+ B = foreach A generate age, registration,contributions;
+ C = order B by contributions;
+ D = limit C 3;
+ E = foreach D generate contributions;
+ STORE C INTO ':OUTPATH:.1';
+ STORE E INTO ':OUTPATH:.2';?,
+ 'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions);
+ B = foreach A generate age, registration,contributions;
+ C = order B by contributions;
+ STORE C INTO ':OUTPATH:.1';
+ F = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions);
+ G = foreach F generate age, registration,contributions;
+ H = order G by contributions;
+ I = limit H 3;
+ J = foreach I generate contributions;
+ STORE J INTO ':OUTPATH:.2';?,
}
],
},{