You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/09/22 22:18:18 UTC
svn commit: r697972 - in /incubator/pig/branches/types: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
test/org/apache/pig/test/TestLogicalPlanBuilder.java
test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
Author: olga
Date: Mon Sep 22 13:18:17 2008
New Revision: 697972
URL: http://svn.apache.org/viewvc?rev=697972&view=rev
Log:
PIG-430: Projections in nested filter and inside foreach do not work
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
Modified: incubator/pig/branches/types/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=697972&r1=697971&r2=697972&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Mon Sep 22 13:18:17 2008
@@ -227,3 +227,6 @@
PIG-411: Pig leaves HOD processes behind if Ctrl-C is used before HOD
connection is fully established (olgan)
+
+ PIG-430: Projections in nested filter and inside foreach do not work (sms
+ via olgan)
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=697972&r1=697971&r2=697972&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Mon Sep 22 13:18:17 2008
@@ -66,6 +66,7 @@
//a map of alias to logical operator for a quick lookup
private Map<String, LogicalOperator> mapAliasOp;
private static Log log = LogFactory.getLog(QueryParser.class);
+ private boolean bracketed = false;
private long getNextId() {
return nodeIdGen.getNextNodeId(scope);
@@ -2084,11 +2085,12 @@
(
<FILTER>
(
- LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
+ LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input)
+| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
| eOp = BaseEvalSpec(over, specs, lp, input)
)
{subSchema = eOp.getSchema();}
- <BY> cond = PCond(subSchema,null,conditionPlan,input)
+ <BY> cond = PCond(subSchema,null,conditionPlan,eOp)
)
{
lp.add(eOp);
@@ -2122,7 +2124,8 @@
(
( <ORDER> | <ARRANGE> )
(
- LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
+ LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input)
+| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
| eOp = BaseEvalSpec(over, specs, lp, input)
)
<BY>
@@ -2184,7 +2187,8 @@
(
<DISTINCT>
(
- LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
+ LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input)
+| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);}
| eOp = BaseEvalSpec(over, specs, lp, input)
)
)
@@ -2981,7 +2985,8 @@
int i;
ExpressionOperator spec = null;
log.trace("Entering BracketedSimpleProj");
- log.debug("eOp: " + eOp.getClass().getName());
+ log.debug("eOp: " + eOp);
+ bracketed = true;
}
{
(
@@ -2989,7 +2994,7 @@
| ("(" spec = SimpleProj(over,lp,eOp) ")")
)
- {log.trace("Exiting BracketedSimpleProj");return spec;}
+ {log.trace("Exiting BracketedSimpleProj");bracketed=false; return spec;}
}
ExpressionOperator SimpleProj(Schema over, LogicalPlan lp, LogicalOperator eOp):
@@ -3189,7 +3194,7 @@
log.debug("eOp: " + eOp.getClass().getName() + " " + eOp);
lp.add(project);
log.debug("DollarVar: Added operator " + project.getClass().getName() + " " + project + " to logical plan " + lp);
- if(eOp instanceof ExpressionOperator) {
+ if((eOp instanceof ExpressionOperator) && (bracketed)) {
lp.add(eOp);
lp.connect(eOp, project);
}
@@ -3273,7 +3278,7 @@
try {
lp.add(item);
log.debug("AliasFieldOrSpec: Added operator " + item.getClass().getName() + " " + item + " to logical plan " + lp);
- if(eOp instanceof ExpressionOperator) {
+ if((eOp instanceof ExpressionOperator) && (bracketed)) {
lp.add(eOp);
lp.connect(eOp, item);
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=697972&r1=697971&r2=697972&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Mon Sep 22 13:18:17 2008
@@ -26,6 +26,7 @@
import java.util.Properties;
import java.net.URL;
import java.util.List;
+import java.util.ArrayList;
import java.util.Set;
import junit.framework.AssertionFailedError;
@@ -1366,6 +1367,46 @@
assertTrue(Schema.equals(foreach.getSchema(), getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray, max_age: double"), false, true));
}
+ @Test
+ public void testQuery96() throws FrontendException, ParseException {
+ buildPlan("a = load 'input' as (name, age, gpa);");
+ buildPlan("b = filter a by age < 20;");
+ buildPlan("c = group b by age;");
+ String query = "d = foreach c {"
+ + "cf = filter b by gpa < 3.0;"
+ + "cd = distinct cf.gpa;"
+ + "co = order cd by $0;"
+ + "generate group, flatten(co);"
+ + "};";
+ LogicalPlan lp = buildPlan(query);
+
+ LOForEach foreach = (LOForEach)lp.getLeaves().get(0);
+ ArrayList<LogicalPlan> foreachPlans = foreach.getForEachPlans();
+ LogicalPlan flattenPlan = foreachPlans.get(1);
+ LogicalOperator project = flattenPlan.getLeaves().get(0);
+ assertTrue(project instanceof LOProject);
+ LogicalOperator sort = flattenPlan.getPredecessors(project).get(0);
+ assertTrue(sort instanceof LOSort);
+ LogicalOperator distinct = flattenPlan.getPredecessors(sort).get(0);
+ assertTrue(distinct instanceof LODistinct);
+
+ //testing the presence of the nested foreach
+ LogicalOperator nestedForeach = flattenPlan.getPredecessors(distinct).get(0);
+ assertTrue(nestedForeach instanceof LOForEach);
+ LogicalPlan nestedForeachPlan = ((LOForEach)nestedForeach).getForEachPlans().get(0);
+ LogicalOperator nestedProject = nestedForeachPlan.getRoots().get(0);
+ assertTrue(nestedProject instanceof LOProject);
+ assertTrue(((LOProject)nestedProject).getCol() == 2);
+
+ //testing the filter inner plan for the absence of the project connected to project
+ LogicalOperator filter = flattenPlan.getPredecessors(nestedForeach).get(0);
+ assertTrue(filter instanceof LOFilter);
+ LogicalPlan comparisonPlan = ((LOFilter)filter).getComparisonPlan();
+ LOLesserThan lessThan = (LOLesserThan)comparisonPlan.getLeaves().get(0);
+ LOProject filterProject = (LOProject)lessThan.getLhsOperand();
+ assertTrue(null == comparisonPlan.getPredecessors(filterProject));
+ }
+
private Schema getSchemaFromString(String schemaString) throws ParseException {
return getSchemaFromString(schemaString, DataType.BYTEARRAY);
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld?rev=697972&r1=697971&r2=697972&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld Mon Sep 22 13:18:17 2008
@@ -6,9 +6,9 @@
| | |
| | Equal To[boolean] - Test-Plan-Builder-14
| | |
-| | |---Project[bytearray][1] - Test-Plan-Builder-12
+| | |---Project[Unknown][1] - Test-Plan-Builder-12
| | |
-| | |---Project[bytearray][2] - Test-Plan-Builder-13
+| | |---Project[Unknown][2] - Test-Plan-Builder-13
| |
| |---Project[tuple][0] - Test-Plan-Builder-10
|