You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pi...@apache.org on 2008/07/16 15:56:33 UTC
svn commit: r677290 - in /incubator/pig/branches/types:
src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
test/org/apache/pig/test/TestLogicalPlanBuilder.java
Author: pisong
Date: Wed Jul 16 06:56:32 2008
New Revision: 677290
URL: http://svn.apache.org/viewvc?rev=677290&view=rev
Log:
PIG-299 Innerplan bag projection to ForEach
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=677290&r1=677289&r2=677290&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Wed Jul 16 06:56:32 2008
@@ -1374,14 +1374,14 @@
{
Token t;
LogicalOperator item;
- ExpressionOperator eOp = null;
+ LogicalOperator eOp = null;
log.trace("Entering NestedCommand");
}
{
(
t = <IDENTIFIER> "="
(
- eOp = InfixExpr(over,specs,lp,input)
+ eOp = NestedProject(over,specs,lp,input)
{
item = eOp;
lp.add(eOp);
@@ -1404,6 +1404,75 @@
}
}
+LogicalOperator NestedProject(Schema over, Map<String, LogicalOperator> specs, LogicalPlan lp, LogicalOperator input):
+{
+ Token t;
+ ArrayList<Integer> colList = new ArrayList<Integer>();
+ int i = -1;
+ LogicalOperator foreachInput = null;
+ Schema subSchema = null;
+ log.trace("Entering NestedFilter");
+}
+{
+ (
+ t = <IDENTIFIER>
+ {
+ if(null != specs) {
+ foreachInput = specs.get(t.image);
+ }
+ if(null == foreachInput) {
+ if ((null == over) || (i = over.getPosition(t.image)) == -1) {
+ throw new ParseException("Invalid alias: " + t.image + " in " + over);
+ }
+ foreachInput = new LOProject(lp, new OperatorKey(scope, getNextId()), input, i);
+ }
+
+ try {
+ lp.add(foreachInput);
+ if(input instanceof ExpressionOperator) {
+ lp.add(input);
+ lp.connect(input, foreachInput);
+ }
+ } catch (Exception planException) {
+ ParseException parseException = new ParseException(planException.getMessage());
+ throw parseException;
+ }
+ }
+| t = <DOLLARVAR>
+ {
+ foreachInput = new LOProject(lp, new OperatorKey(scope, getNextId()), input, undollar(t.image));
+ }
+ )
+ {
+ if(foreachInput instanceof ExpressionOperator) {
+ subSchema = ((ExpressionOperator)foreachInput).getFieldSchema().schema;
+ } else {
+ subSchema = foreachInput.getSchema();
+ }
+ }
+ "."
+ (
+ i = ColName(subSchema) {colList.add(i);}
+| "(" i = ColName(subSchema) {colList.add(i);} ("," i = ColName(over) {colList.add(i);})* ")"
+ )
+ {
+ ArrayList<LogicalPlan> foreachPlans = new ArrayList<LogicalPlan>();
+ ArrayList<Boolean> flattenList = new ArrayList<Boolean>();
+ for(int j: colList) {
+ LogicalPlan plan = new LogicalPlan();
+ LOProject project = new LOProject(plan, new OperatorKey(scope, getNextId()), foreachInput, j);
+ plan.add(project);
+ foreachPlans.add(plan);
+ flattenList.add(false);
+ }
+ LogicalOperator foreach = new LOForEach(lp, new OperatorKey(scope, getNextId()), foreachPlans, flattenList);
+ lp.add(foreach);
+ lp.add(foreachInput);
+ lp.connect(foreachInput, foreach);
+ return foreach;
+ }
+}
+
LogicalOperator NestedFilter(Schema over, Map<String, LogicalOperator> specs, LogicalPlan lp, LogicalOperator input):
{
ExpressionOperator cond;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=677290&r1=677289&r2=677290&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Wed Jul 16 06:56:32 2008
@@ -923,7 +923,7 @@
+ "generate group, flatten(co);"
//+ "generate group, flatten(cd);"
+ "};";
- printPlan(buildPlan(query));
+ buildPlan(query);
}
@Test
@@ -942,6 +942,63 @@
}
}
+ @Test
+ public void testQuery82() {
+ buildPlan("a = load 'myfile';");
+ buildPlan("b = group a by $0;");
+ String query = "c = foreach b {"
+ + "c1 = order $1 by *;"
+ + "c2 = $1.$0;"
+ + "generate flatten(c1), c2;"
+ + "};";
+ buildPlan(query);
+ }
+
+ @Test
+ public void testQueryFail82() {
+ buildPlan("a = load 'myfile';");
+ buildPlan("b = group a by $0;");
+ String query = "c = foreach b {"
+ + "c1 = order $1 by *;"
+ + "c2 = $1;"
+ + "generate flatten(c1), c2;"
+ + "};";
+ try {
+ buildPlan(query);
+ } catch (AssertionFailedError e) {
+ assertTrue(e.getMessage().contains("Exception"));
+ }
+ }
+
+ @Test
+ public void testQuery83() {
+ buildPlan("a = load 'input1' as (name, age, gpa);");
+ buildPlan("b = filter a by age < '20';");
+ buildPlan("c = group b by (name,age);");
+ String query = "d = foreach c {"
+ + "cf = filter b by gpa < '3.0';"
+ + "cp = cf.gpa;"
+ + "cd = distinct cp;"
+ + "co = order cd by gpa;"
+ + "generate group, flatten(co);"
+ + "};";
+ buildPlan(query);
+ }
+
+ @Test
+ public void testQuery84() {
+ buildPlan("a = load 'input1' as (name, age, gpa);");
+ buildPlan("b = filter a by age < '20';");
+ buildPlan("c = group b by (name,age);");
+ String query = "d = foreach c {"
+ + "cf = filter b by gpa < '3.0';"
+ + "cp = cf.$2;"
+ + "cd = distinct cp;"
+ + "co = order cd by gpa;"
+ + "generate group, flatten(co);"
+ + "};";
+ buildPlan(query);
+ }
private void printPlan(LogicalPlan lp) {
LOPrinter graphPrinter = new LOPrinter(System.err, lp);