You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/09/27 07:27:53 UTC
svn commit: r1176199 - in /pig/branches/branch-0.9: ./ src/org/apache/pig/
src/org/apache/pig/parser/ test/org/apache/pig/parser/
test/org/apache/pig/test/
Author: thejas
Date: Tue Sep 27 05:27:52 2011
New Revision: 1176199
URL: http://svn.apache.org/viewvc?rev=1176199&view=rev
Log:
PIG-2223: error accessing column in output schema of udf having project-star input (thejas)
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/branches/branch-0.9/test/org/apache/pig/parser/TestColumnAliasConversion.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestDataBagAccess.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Sep 27 05:27:52 2011
@@ -40,6 +40,8 @@ PIG-2221: Couldnt find documentation for
BUG FIXES
+PIG-2223: error accessing column in output schema of udf having project-star input (thejas)
+
PIG-2299: jetty 6.1.14 startup issue causes unit tests to fail in CI (thw via daijy)
PIG-2301: Some more bin/pig, build.xml cleanup for 0.9.1 (daijy)
Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Tue Sep 27 05:27:52 2011
@@ -82,6 +82,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.AllExpressionVisitor;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
@@ -1654,7 +1655,6 @@ public class PigServer {
}
private void compile(LogicalPlan lp) throws FrontendException {
- new ProjStarInUdfExpander(lp).visit();
new ColumnAliasConversionVisitor( lp ).visit();
new SchemaAliasVisitor( lp ).visit();
new ScalarVisitor( lp, pigContext ).visit();
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Sep 27 05:27:52 2011
@@ -54,13 +54,17 @@ import org.apache.pig.impl.streaming.Str
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec;
import org.apache.pig.impl.util.MultiMap;
import org.apache.pig.impl.util.StringUtils;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.logical.expression.ConstantExpression;
import org.apache.pig.newplan.logical.expression.LessThanEqualExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOCogroup.GROUPTYPE;
import org.apache.pig.newplan.logical.relational.LOCross;
import org.apache.pig.newplan.logical.relational.LODistinct;
import org.apache.pig.newplan.logical.relational.LOFilter;
@@ -68,6 +72,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LOGenerate;
import org.apache.pig.newplan.logical.relational.LOInnerLoad;
import org.apache.pig.newplan.logical.relational.LOJoin;
+import org.apache.pig.newplan.logical.relational.LOJoin.JOINTYPE;
import org.apache.pig.newplan.logical.relational.LOLimit;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LONative;
@@ -80,12 +85,9 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
-import org.apache.pig.newplan.logical.relational.LOCogroup.GROUPTYPE;
-import org.apache.pig.newplan.logical.relational.LOJoin.JOINTYPE;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.apache.pig.newplan.logical.visitor.ProjStarInUdfExpander;
import org.apache.pig.newplan.logical.visitor.ProjectStarExpander;
-import org.apache.pig.newplan.Operator;
-import org.apache.pig.newplan.OperatorPlan;
public class LogicalPlanBuilder {
private LogicalPlan plan = new LogicalPlan();
@@ -148,9 +150,18 @@ public class LogicalPlanBuilder {
return new LOFilter( plan );
}
- String buildFilterOp(SourceLocation loc, LOFilter op, String alias, String inputAlias, LogicalExpressionPlan expr) throws ParserValidationException {
+ String buildFilterOp(SourceLocation loc, LOFilter op, String alias,
+ String inputAlias, LogicalExpressionPlan expr)
+ throws ParserValidationException {
op.setFilterPlan( expr );
- return buildOp( loc, op, alias, inputAlias, null );
+ alias = buildOp( loc, op, alias, inputAlias, null ); // it should actually return same alias
+ try {
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan()).visit(op);
+ } catch (FrontendException e) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
+ return alias;
}
String buildDistinctOp(SourceLocation loc, String alias, String inputAlias, String partitioner) throws ParserValidationException {
@@ -164,7 +175,9 @@ public class LogicalPlanBuilder {
}
String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
- SourceLocation valLoc) throws ParserValidationException {
+ SourceLocation valLoc)
+ throws ParserValidationException {
+
LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
// Generate a filter condition.
LogicalExpression konst = new ConstantExpression( filterPlan, value);
@@ -216,6 +229,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, sort, alias, inputAlias, null );
try {
(new ProjectStarExpander(sort.getPlan())).visit(sort);
+ (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
+ new SchemaResetter(sort.getPlan()).visit(sort);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -275,6 +290,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAliases, partitioner );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan()).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -316,6 +333,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAliases, partitioner );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan()).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -429,6 +448,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAlias, null );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan()).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestColumnAliasConversion.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestColumnAliasConversion.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestColumnAliasConversion.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestColumnAliasConversion.java Tue Sep 27 05:27:52 2011
@@ -101,7 +101,7 @@ public class TestColumnAliasConversion {
"C = store B into 'output';";
try {
validate( query );
- } catch(PlanValidationException ex) {
+ }catch(ParserValidationException ex){
return;
}
Assert.fail( "Query should fail to validate." );
@@ -140,7 +140,7 @@ public class TestColumnAliasConversion {
"C = store B into 'output';";
try {
validate( query );
- } catch(PlanValidationException ex) {
+ } catch(ParserValidationException ex) {
return;
}
Assert.fail( "Query should fail to validate." );
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestDataBagAccess.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestDataBagAccess.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestDataBagAccess.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestDataBagAccess.java Tue Sep 27 05:27:52 2011
@@ -130,7 +130,7 @@ public class TestDataBagAccess extends T
} catch(FrontendException e) {
exceptionOccured = true;
String msg = e.getMessage();
- Util.checkStrContainsSubStr(msg, "Invalid field reference. Referenced field [t] does not exist in schema");
+ Util.checkStrContainsSubStr(msg, "Cannot find field t in i:int,d:double,c:chararray");
}
assertTrue(exceptionOccured);
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java Tue Sep 27 05:27:52 2011
@@ -898,8 +898,8 @@ public class TestEvalPipeline2 {
pigServer.openIterator("c");
} catch (Exception e) {
PigException pe = LogUtils.getPigException(e);
- Assert.assertTrue(pe.getErrorCode()==1031);
- Assert.assertTrue(pe.getMessage().contains("Incompatable schema"));
+ Util.checkStrContainsSubStr(pe.getMessage(), "ERROR 1031");
+ Util.checkStrContainsSubStr(pe.getMessage(), "Incompatable schema");
return;
}
Assert.fail();
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1176199&r1=1176198&r2=1176199&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Tue Sep 27 05:27:52 2011
@@ -145,6 +145,33 @@ public class TestProjectStarRangeInUdf
}
+ //PIG-2223
+ // lookup on column name in udf output tuple schema
+ @Test
+ public void testProjStarExpandInForeachLookup1() throws IOException {
+
+ String query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+ + "f = foreach l1 generate TOTUPLE(*) as tb;"
+ + "f2 = foreach f generate tb.a, tb.b;"
+ ;
+ compileAndCompareSchema("a : int, b : int", query, "f2");
+ }
+
+ //PIG-2223
+ // lookup on column name in udf output tuple schema
+ @Test
+ public void testProjStarExpandInForeachLookup2() throws IOException {
+
+ String query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+ + "f = foreach l1 generate TOTUPLE(b .. ) as tb;"
+ + "f2 = foreach f generate tb.b as b2, tb.c as c2;"
+ + "f3 = foreach f2 generate b2, b2 + c2 as bc2;"
+ ;
+ compileAndCompareSchema("b2 : int, bc2 : int", query, "f3");
+ }
+
@Test
public void testProjStarExpandInFilter1() throws IOException{
//TOBAG has * and a bincond expression as argument