You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/09/27 07:28:53 UTC
svn commit: r1176200 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/parser/ test/org/apache/pig/parser/
test/org/apache/pig/test/
Author: thejas
Date: Tue Sep 27 05:28:52 2011
New Revision: 1176200
URL: http://svn.apache.org/viewvc?rev=1176200&view=rev
Log:
PIG-2223: error accessing column in output schema of udf having project-star input (thejas)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java
pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 27 05:28:52 2011
@@ -134,6 +134,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2223: error accessing column in output schema of udf having project-star input (thejas)
+
PIG-2273: Pig.compileFromFile in embedded python fails when pig script starts with a comment (ddaniels888 via gates)
PIG-2278: Wrong version numbers for libraries in eclipse template classpath (azaroth)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Tue Sep 27 05:28:52 2011
@@ -83,6 +83,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.AllExpressionVisitor;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
@@ -1613,7 +1614,6 @@ public class PigServer {
}
private void compile(LogicalPlan lp) throws FrontendException {
- new ProjStarInUdfExpander(lp).visit();
new ColumnAliasConversionVisitor( lp ).visit();
new SchemaAliasVisitor( lp ).visit();
new ScalarVisitor( lp, pigContext ).visit();
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Sep 27 05:28:52 2011
@@ -63,6 +63,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOCogroup.GROUPTYPE;
import org.apache.pig.newplan.logical.relational.LOCross;
@@ -87,6 +88,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
import org.apache.pig.newplan.logical.rules.OptimizerUtils;
+import org.apache.pig.newplan.logical.visitor.ProjStarInUdfExpander;
import org.apache.pig.newplan.logical.visitor.ProjectStarExpander;
public class LogicalPlanBuilder {
@@ -158,9 +160,19 @@ public class LogicalPlanBuilder {
return new LOFilter( plan, true );
}
- String buildFilterOp(SourceLocation loc, LOFilter op, String alias, String inputAlias, LogicalExpressionPlan expr) throws ParserValidationException {
+ String buildFilterOp(SourceLocation loc, LOFilter op, String alias,
+ String inputAlias, LogicalExpressionPlan expr)
+ throws ParserValidationException {
+
op.setFilterPlan( expr );
- return buildOp( loc, op, alias, inputAlias, null );
+ alias = buildOp( loc, op, alias, inputAlias, null ); // it should actually return same alias
+ try {
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
+ } catch (FrontendException e) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
+ return alias;
}
String buildDistinctOp(SourceLocation loc, String alias, String inputAlias, String partitioner) throws ParserValidationException {
@@ -179,7 +191,9 @@ public class LogicalPlanBuilder {
}
String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
- SourceLocation valLoc) throws ParserValidationException {
+ SourceLocation valLoc)
+ throws ParserValidationException {
+
LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
// Generate a filter condition.
LogicalExpression konst = new ConstantExpression( filterPlan, value);
@@ -191,7 +205,9 @@ public class LogicalPlanBuilder {
}
String buildSampleOp(SourceLocation loc, LOFilter filter, String alias, String inputAlias,
- LogicalExpressionPlan samplePlan, LogicalExpression expr) throws ParserValidationException {
+ LogicalExpressionPlan samplePlan, LogicalExpression expr)
+ throws ParserValidationException {
+
UserFuncExpression udf = new UserFuncExpression( samplePlan, new FuncSpec( RANDOM.class.getName() ) );
new LessThanExpression( samplePlan, udf, expr );
return buildFilterOp( loc, filter, alias, inputAlias, samplePlan );
@@ -280,6 +296,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, sort, alias, inputAlias, null );
try {
(new ProjectStarExpander(sort.getPlan())).visit(sort);
+ (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
+ new SchemaResetter(sort.getPlan(), true).visit(sort);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -339,6 +357,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAliases, partitioner );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -380,6 +400,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAliases, partitioner );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
@@ -494,6 +516,8 @@ public class LogicalPlanBuilder {
alias = buildOp( loc, op, alias, inputAlias, null );
try {
(new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
} catch (FrontendException e) {
throw new ParserValidationException( intStream, loc, e );
}
Modified: pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java Tue Sep 27 05:28:52 2011
@@ -101,7 +101,7 @@ public class TestColumnAliasConversion {
"C = store B into 'output';";
try {
validate( query );
- } catch(PlanValidationException ex) {
+ }catch(ParserValidationException ex){
return;
}
Assert.fail( "Query should fail to validate." );
@@ -140,7 +140,7 @@ public class TestColumnAliasConversion {
"C = store B into 'output';";
try {
validate( query );
- } catch(PlanValidationException ex) {
+ } catch(ParserValidationException ex) {
return;
}
Assert.fail( "Query should fail to validate." );
Modified: pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java Tue Sep 27 05:28:52 2011
@@ -130,7 +130,7 @@ public class TestDataBagAccess extends T
} catch(FrontendException e) {
exceptionOccured = true;
String msg = e.getMessage();
- Util.checkStrContainsSubStr(msg, "Invalid field reference. Referenced field [t] does not exist in schema");
+ Util.checkStrContainsSubStr(msg, "Cannot find field t in i:int,d:double,c:chararray");
}
assertTrue(exceptionOccured);
}
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Tue Sep 27 05:28:52 2011
@@ -898,8 +898,8 @@ public class TestEvalPipeline2 {
pigServer.openIterator("c");
} catch (Exception e) {
PigException pe = LogUtils.getPigException(e);
- Assert.assertTrue(pe.getErrorCode()==1031);
- Assert.assertTrue(pe.getMessage().contains("Incompatable schema"));
+ Util.checkStrContainsSubStr(pe.getMessage(), "ERROR 1031");
+ Util.checkStrContainsSubStr(pe.getMessage(), "Incompatable schema");
return;
}
Assert.fail();
Modified: pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Tue Sep 27 05:28:52 2011
@@ -145,6 +145,33 @@ public class TestProjectStarRangeInUdf
}
+ //PIG-2223
+ // lookup on column name in udf output tuple schema
+ @Test
+ public void testProjStarExpandInForeachLookup1() throws IOException {
+
+ String query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+ + "f = foreach l1 generate TOTUPLE(*) as tb;"
+ + "f2 = foreach f generate tb.a, tb.b;"
+ ;
+ compileAndCompareSchema("a : int, b : int", query, "f2");
+ }
+
+ //PIG-2223
+ // lookup on column name in udf output tuple schema
+ @Test
+ public void testProjStarExpandInForeachLookup2() throws IOException {
+
+ String query =
+ " l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+ + "f = foreach l1 generate TOTUPLE(b .. ) as tb;"
+ + "f2 = foreach f generate tb.b as b2, tb.c as c2;"
+ + "f3 = foreach f2 generate b2, b2 + c2 as bc2;"
+ ;
+ compileAndCompareSchema("b2 : int, bc2 : int", query, "f3");
+ }
+
@Test
public void testProjStarExpandInFilter1() throws IOException{
//TOBAG has * and a bincond expression as argument