You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/12/11 04:52:02 UTC
svn commit: r1044563 - in /pig/branches/branch-0.8: CHANGES.txt
src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java
src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java
test/org/apache/pig/test/TestFilterSimplification.java
Author: yanz
Date: Sat Dec 11 03:52:01 2010
New Revision: 1044563
URL: http://svn.apache.org/viewvc?rev=1044563&view=rev
Log:
Logical simplification fails on map key referenced values (yanz)
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java
pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java
pig/branches/branch-0.8/test/org/apache/pig/test/TestFilterSimplification.java
Modified: pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1044563&r1=1044562&r2=1044563&view=diff
==============================================================================
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Sat Dec 11 03:52:01 2010
@@ -211,6 +211,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1762: Logical simplification fails on map key referenced values (yanz)
+
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
PIG-1760: Need to report progress in all databags (rding)
Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java?rev=1044563&r1=1044562&r2=1044563&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java (original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java Sat Dec 11 03:52:01 2010
@@ -58,8 +58,22 @@ public class MapLookupExpression extends
public boolean isEqual(Operator other) throws FrontendException {
if (other != null && other instanceof MapLookupExpression) {
MapLookupExpression po = (MapLookupExpression)other;
- return ( po.mMapKey.compareTo(mMapKey) == 0 ) &&
- po.mValueSchema.isEqual( mValueSchema );
+ if ( po.mMapKey.compareTo(mMapKey) != 0 ||
+ !po.mValueSchema.isEqual( mValueSchema ))
+ return false;
+ else {
+ // check the nested map equality
+ if (plan.getSuccessors(this) != null) {
+ if (other.getPlan().getSuccessors(other) == null)
+ return false;
+ else {
+ return plan.getSuccessors(this).get(0).isEqual(other.getPlan().getSuccessors(other).get(0));
+ }
+ } else if (other.getPlan().getSuccessors(other) != null) {
+ return false;
+ } else
+ return true;
+ }
} else {
return false;
}
Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java?rev=1044563&r1=1044562&r2=1044563&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java (original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java Sat Dec 11 03:52:01 2010
@@ -33,7 +33,7 @@ import org.apache.pig.newplan.OperatorPl
* generated.
*
*/
-class DNFPlanGenerator extends AllSameExpressionVisitor {
+class DNFPlanGenerator extends LogicalExpressionVisitor {
private OperatorPlan dnfPlan = null;
Stack<LogicalExpression> result;
@@ -42,34 +42,15 @@ class DNFPlanGenerator extends AllSameEx
result = new Stack<LogicalExpression>();
}
- @Override
- protected void execute(LogicalExpression op)
- throws FrontendException {
- if (op instanceof UnaryExpression) {
- result.pop();
- }
- else if (op instanceof BinaryExpression) {
- result.pop();
- result.pop();
- }
- else if (op instanceof BinCondExpression) {
- result.pop();
- result.pop();
- result.pop();
- }
-
- result.push(op);
- }
-
OperatorPlan getDNFPlan() {
- if (dnfPlan == null) dnfPlan = result.pop().getPlan();
+ if (dnfPlan == null) dnfPlan = (result.empty() ? plan : result.pop().getPlan());
return dnfPlan;
}
@Override
public void visit(AndExpression exp) throws FrontendException {
- LogicalExpression rhsExp = result.pop();
- LogicalExpression lhsExp = result.pop();
+ LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs() instanceof OrExpression ? result.pop() : exp.getRhs()));
+ LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs() instanceof OrExpression ? result.pop() : exp.getLhs()));
if (!(lhsExp instanceof AndExpression) && !(lhsExp instanceof DNFExpression) && !(lhsExp instanceof OrExpression) && !(rhsExp instanceof AndExpression) && !(rhsExp instanceof OrExpression) && !(rhsExp instanceof DNFExpression)) result.push(exp);
else {
if (dnfPlan == null) dnfPlan = new DNFPlan();
@@ -368,8 +349,8 @@ class DNFPlanGenerator extends AllSameEx
@Override
public void visit(OrExpression exp) throws FrontendException {
- LogicalExpression rhsExp = result.pop();
- LogicalExpression lhsExp = result.pop();
+ LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs() instanceof OrExpression ? result.pop() : exp.getRhs()));
+ LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs() instanceof OrExpression ? result.pop() : exp.getLhs()));
if (!(lhsExp instanceof OrExpression) &&
(!(lhsExp instanceof DNFExpression) ||
((DNFExpression) lhsExp).type == DNFExpression.DNFExpressionType.AND) && !(rhsExp instanceof OrExpression) && (!(rhsExp instanceof DNFExpression) || ((DNFExpression) rhsExp).type == DNFExpression.DNFExpressionType.AND)) result.push(exp);
Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestFilterSimplification.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestFilterSimplification.java?rev=1044563&r1=1044562&r2=1044563&view=diff
==============================================================================
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestFilterSimplification.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestFilterSimplification.java Sat Dec 11 03:52:01 2010
@@ -770,6 +770,40 @@ public class TestFilterSimplification ex
assertTrue(expected.isEqual(newLogicalPlan));
}
+ @Test
+ public void test7() throws Exception {
+ LogicalPlanTester lpt = new LogicalPlanTester(pc);
+ lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);");
+
+ org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
+ LogicalPlan newLogicalPlan = migratePlan(plan);
+
+ PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+ optimizer.optimize();
+
+ lpt = new LogicalPlanTester(pc);
+ lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);");
+ plan = lpt.buildPlan("store b into 'empty';");
+ LogicalPlan expected = migratePlan(plan);
+
+ assertTrue(expected.isEqual(newLogicalPlan));
+
+ lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);");
+
+ plan = lpt.buildPlan("store b into 'empty';");
+ newLogicalPlan = migratePlan(plan);
+
+ optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+ optimizer.optimize();
+
+ lpt = new LogicalPlanTester(pc);
+ lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);");
+ plan = lpt.buildPlan("store b into 'empty';");
+ expected = migratePlan(plan);
+
+ assertTrue(expected.isEqual(newLogicalPlan));
+ }
+
public class MyPlanOptimizer extends LogicalPlanOptimizer {
protected MyPlanOptimizer(OperatorPlan p, int iterations) {