You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/04/21 00:44:21 UTC
svn commit: r1095539 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Author: thejas
Date: Wed Apr 20 22:44:21 2011
New Revision: 1095539
URL: http://svn.apache.org/viewvc?rev=1095539&view=rev
Log:
PIG-1281: Detect org.apache.pig.data.DataByteArray cannot be cast to
org.apache.pig.data.Tuple type of errors at Compile Type during
creation of logical plan (thejas)
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1095539&r1=1095538&r2=1095539&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Wed Apr 20 22:44:21 2011
@@ -158,6 +158,10 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1281: Detect org.apache.pig.data.DataByteArray cannot be cast to
+ org.apache.pig.data.Tuple type of errors at Compile Type during
+ creation of logical plan (thejas)
+
PIG-1939: order-by statement should support project-range to-end in
any position among the sort columns if input schema is known (thejas)
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java?rev=1095539&r1=1095538&r2=1095539&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java Wed Apr 20 22:44:21 2011
@@ -51,6 +51,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.BinaryExpression;
import org.apache.pig.newplan.logical.expression.CastExpression;
import org.apache.pig.newplan.logical.expression.ConstantExpression;
+import org.apache.pig.newplan.logical.expression.DereferenceExpression;
import org.apache.pig.newplan.logical.expression.DivideExpression;
import org.apache.pig.newplan.logical.expression.EqualExpression;
import org.apache.pig.newplan.logical.expression.GreaterThanEqualExpression;
@@ -646,6 +647,24 @@ public class TypeCheckingExpVisitor exte
}
map.resetFieldSchema();
}
+
+ @Override
+ public void visit(DereferenceExpression deref) throws FrontendException{
+ byte inputType = deref.getReferredExpression().getType();
+ switch(inputType){
+ case DataType.TUPLE:
+ case DataType.BAG:
+ case DataType.BYTEARRAY: // ideally determine type at runtime
+ //allowed types
+ break;
+ default:
+ int errCode = 1129;
+ String msg = "Referring to column(s) within a column of type " +
+ DataType.findTypeName(inputType)
+ + " is not allowed";
+ throw new TypeCheckerException(deref, msg, errCode, PigException.INPUT);
+ }
+ }
@Override
public void visit(UserFuncExpression func) throws FrontendException{
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1095539&r1=1095538&r2=1095539&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Wed Apr 20 22:44:21 2011
@@ -65,7 +65,6 @@ import org.apache.pig.newplan.OperatorPl
import org.apache.pig.newplan.PlanVisitor;
import org.apache.pig.newplan.PlanWalker;
import org.apache.pig.newplan.ReverseDependencyOrderWalker;
-import org.apache.pig.newplan.logical.Util;
import org.apache.pig.newplan.logical.expression.AddExpression;
import org.apache.pig.newplan.logical.expression.AndExpression;
import org.apache.pig.newplan.logical.expression.BinCondExpression;
@@ -527,7 +526,7 @@ public class TestTypeCheckingValidatorNe
ConstantExpression constant1 = new ConstantExpression(plan, innerTuple) ;
ConstantExpression constant2 = new ConstantExpression(plan, tuple) ;
CastExpression cast1 = new CastExpression(plan, constant1,
- Util.translateFieldSchema(new FieldSchema(null, castSchema, DataType.TUPLE))) ;
+ org.apache.pig.newplan.logical.Util.translateFieldSchema(new FieldSchema(null, castSchema, DataType.TUPLE))) ;
EqualExpression equal1 = new EqualExpression(plan, cast1, constant2) ;
@@ -605,7 +604,7 @@ public class TestTypeCheckingValidatorNe
ConstantExpression constant1 = new ConstantExpression(plan, innerTuple) ;
ConstantExpression constant2 = new ConstantExpression(plan, tuple) ;
CastExpression cast1 = new CastExpression(plan, constant1,
- Util.translateFieldSchema(new FieldSchema(null, castSchema, DataType.TUPLE))) ;
+ org.apache.pig.newplan.logical.Util.translateFieldSchema(new FieldSchema(null, castSchema, DataType.TUPLE))) ;
EqualExpression equal1 = new EqualExpression(plan, cast1, constant2) ;
CompilationMessageCollector collector = new CompilationMessageCollector() ;
@@ -835,8 +834,8 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema(inputSchema1));
- load2.setScriptSchema(Util.translateSchema(inputSchema2));
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema2));
// create union operator
ArrayList<LogicalRelationalOperator> inputList = new ArrayList<LogicalRelationalOperator>() ;
@@ -859,7 +858,7 @@ public class TestTypeCheckingValidatorNe
printMessageCollector(collector) ;
// check end result schema
- Schema outputSchema = Util.translateSchema(union.getSchema()) ;
+ Schema outputSchema = org.apache.pig.newplan.logical.Util.translateSchema(union.getSchema()) ;
Schema expectedSchema = null ;
{
@@ -957,8 +956,8 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema(inputSchema1));
- load2.setScriptSchema(Util.translateSchema(inputSchema2));
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema2));
@@ -984,7 +983,7 @@ public class TestTypeCheckingValidatorNe
printMessageCollector(collector) ;
// check end result schema
- Schema outputSchema = Util.translateSchema(union.getSchema()) ;
+ Schema outputSchema = org.apache.pig.newplan.logical.Util.translateSchema(union.getSchema()) ;
Schema expectedSchema = null ;
{
@@ -1059,7 +1058,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setSchema(Util.translateSchema(inputSchema1)) ;
+ load1.setSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1)) ;
// create union operator
LODistinct distinct1 = new LODistinct(plan) ;
@@ -1119,7 +1118,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema(inputSchema1)) ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1)) ;
// Create inner plan
LogicalExpressionPlan innerPlan = new LogicalExpressionPlan() ;
@@ -1183,7 +1182,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
// Create inner plan
LogicalExpressionPlan innerPlan = new LogicalExpressionPlan() ;
@@ -1246,7 +1245,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
// Create project inner plan #1
LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan() ;
@@ -1335,7 +1334,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
LOSort sort1 = new LOSort(plan) ;
@@ -1421,7 +1420,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
// Sort
@@ -1503,7 +1502,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
// split
@@ -1610,7 +1609,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
// split
LOSplit split1 = new LOSplit(plan);
@@ -1708,9 +1707,9 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ;
- load2.setScriptSchema(Util.translateSchema((inputSchema2))) ;
- load2.setScriptSchema(Util.translateSchema((inputSchema2))) ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
LOCogroup cogroup1 = new LOCogroup(plan);
@@ -1845,8 +1844,8 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
- load2.setScriptSchema(Util.translateSchema((inputSchema2))) ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
LOCogroup cogroup1 = new LOCogroup(plan);
@@ -1965,8 +1964,8 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ;
- load2.setScriptSchema(Util.translateSchema((inputSchema2))) ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ;
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema2))) ;
LOCogroup cogroup1 = new LOCogroup(plan);
// Create expression inner plan #1
@@ -2061,7 +2060,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
LogicalPlan innerRelPlan = new LogicalPlan();
LOForEach foreach1 = new LOForEach(plan);
@@ -2154,7 +2153,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
LogicalPlan innerRelPlan = new LogicalPlan();
LOForEach foreach1 = new LOForEach(plan);
@@ -2251,7 +2250,7 @@ public class TestTypeCheckingValidatorNe
}
// set schemas
- load1.setScriptSchema(Util.translateSchema((inputSchema1))) ; ;
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1))) ; ;
LogicalPlan innerRelPlan = new LogicalPlan();
LOForEach foreach1 = new LOForEach(plan);
foreach1.setInnerPlan(innerRelPlan);
@@ -2351,8 +2350,8 @@ public class TestTypeCheckingValidatorNe
Schema schema2 = genFlatSchema(aliases2, types2) ;
// set schemas
- load1.setScriptSchema(Util.translateSchema(schema1)) ;
- load2.setScriptSchema(Util.translateSchema(schema2));
+ load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(schema1)) ;
+ load2.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(schema2));
LOCross cross = new LOCross(plan) ;
@@ -3738,7 +3737,7 @@ public class TestTypeCheckingValidatorNe
}
Schema expectedSchema = new Schema(bagFs);
- Schema foreachSch = Util.translateSchema(foreach.getSchema());
+ Schema foreachSch = org.apache.pig.newplan.logical.Util.translateSchema(foreach.getSchema());
assertTrue(Schema.equals(foreachSch, expectedSchema, false, true));
}
@@ -3758,7 +3757,7 @@ public class TestTypeCheckingValidatorNe
"A::gpa: float,B::name: chararray,B::age: int," +
"B::registration: chararray,B::contributions: float";
Schema expectedSchema = org.apache.pig.test.Util.getSchemaFromString(expectedSchemaString);
- Schema foreachSch = Util.translateSchema(foreach.getSchema());
+ Schema foreachSch = org.apache.pig.newplan.logical.Util.translateSchema(foreach.getSchema());
assertTrue(Schema.equals(foreachSch, expectedSchema, false, true));
}
@@ -3940,6 +3939,29 @@ public class TestTypeCheckingValidatorNe
LogicalPlan plan = createAndProcessLPlan(query);
checkLoaderInCasts(plan, PigStorage.class.getName());
}
+
+
+ @Test
+ public void testColumnWithinNonTupleBag() throws IOException {
+ {
+ String query =
+ " l = load 'x' as (i : int);" +
+ " f = foreach l generate i.$0; ";
+ Util.checkExceptionMessage(query, "f",
+ "Referring to column(s) within a column of type " +
+ "int is not allowed"
+ );
+ }
+ {
+ String query =
+ " l = load 'x' as (i : map[]);" +
+ " f = foreach l generate i.$0; ";
+ Util.checkExceptionMessage(query, "f",
+ "Referring to column(s) within a column of type " +
+ "map is not allowed"
+ );
+ }
+ }
// See PIG-1929
@Test