You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/10/17 22:32:00 UTC
svn commit: r705745 - in /incubator/pig/branches/types: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/LOForEach.java
src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
test/org/apache/pig/test/TestTypeCheckingValidator.java
Author: olga
Date: Fri Oct 17 13:31:59 2008
New Revision: 705745
URL: http://svn.apache.org/viewvc?rev=705745&view=rev
Log:
Load Func for POCast is not being set in some cases
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
Modified: incubator/pig/branches/types/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=705745&r1=705744&r2=705745&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Fri Oct 17 13:31:59 2008
@@ -286,3 +286,8 @@
PIG-468: make determine Schema work for BinStorage (pradeepk via olgan)
PIG-494: invalid handling of UTF-8 data in PigStorage (pradeepk via olgan)
+
+ PIG-495: projecting of bags only give 1 field (olgan)
+
+ PIG-500: Load Func for POCast is not being set in some cases (sms via
+ olgan)
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=705745&r1=705744&r2=705745&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java Fri Oct 17 13:31:59 2008
@@ -294,17 +294,19 @@
//just populate the schema with the field schema of the expression operator
//check if the user has defined a schema for the operator; compare the schema
//with that of the expression operator field schema and then add it to the list
+ Schema.FieldSchema newFs = new Schema.FieldSchema(planFs);
if(null != userDefinedSchema) {
try {
- planFs = planFs.mergePrefixFieldSchema(userDefinedSchema.getField(0));
- updateAliasCount(aliases, planFs.alias);
+ newFs = newFs.mergePrefixFieldSchema(userDefinedSchema.getField(0));
+ updateAliasCount(aliases, newFs.alias);
} catch (SchemaMergeException sme) {
throw new FrontendException(sme.getMessage());
} catch (ParseException pe) {
throw new FrontendException(pe.getMessage());
}
}
- fss.add(planFs);
+ newFs.setParent(planFs.canonicalName, op);
+ fss.add(newFs);
}
} else {
//did not get a valid list of field schemas
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=705745&r1=705744&r2=705745&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Fri Oct 17 13:31:59 2008
@@ -1616,11 +1616,20 @@
if(inputType == DataType.BYTEARRAY) {
try {
LoadFunc loadFunc = getLoadFunc(cast.getExpression());
+ if((null == loadFunc) && (expectedType != DataType.BYTEARRAY)) {
+ String msg = "Internal error. Could not resolve load function to use for casting from " +
+ DataType.findTypeName(inputType) + " to " +
+ DataType.findTypeName(expectedType) + ". Found null.";
+ msgCollector.collect(msg, MessageType.Error);
+ throw new VisitorException(msg);
+ }
cast.setLoadFunc(loadFunc);
} catch (FrontendException fee) {
- throw new VisitorException("Cannot resolve load function to use for casting from " +
+ String msg = "Cannot resolve load function to use for casting from " +
DataType.findTypeName(inputType) + " to " +
- DataType.findTypeName(expectedType) + ". " + fee.getMessage());
+ DataType.findTypeName(expectedType) + ". " + fee.getMessage();
+ msgCollector.collect(msg, MessageType.Error);
+ throw new VisitorException(msg, fee);
}
}
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=705745&r1=705744&r2=705745&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java Fri Oct 17 13:31:59 2008
@@ -156,7 +156,7 @@
}
@Test
- public void testExpressionTypeChecking2() throws Throwable {
+ public void testExpressionTypeChecking2Fail() throws Throwable {
LogicalPlan plan = new LogicalPlan() ;
LOConst constant1 = new LOConst(plan, genNewOperatorKey(), 10) ;
constant1.setType(DataType.INTEGER) ;
@@ -193,23 +193,20 @@
CompilationMessageCollector collector = new CompilationMessageCollector() ;
TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
- typeValidator.validate(plan, collector) ;
+
+ try {
+ typeValidator.validate(plan, collector) ;
+ } catch (PlanValidationException pve) {
+ // good
+ }
+
printMessageCollector(collector) ;
printTypeGraph(plan) ;
- if (collector.hasError()) {
- throw new Exception("Error during type checking") ;
+ if (!collector.hasError()) {
+ throw new Exception("Error expected during type checking") ;
}
- // Induction check
- assertEquals(DataType.INTEGER, sub1.getType()) ;
- assertEquals(DataType.BOOLEAN, gt1.getType()) ;
- assertEquals(DataType.BOOLEAN, and1.getType()) ;
- assertEquals(DataType.BOOLEAN, not1.getType()) ;
-
- // Cast insertion check
- assertEquals(DataType.INTEGER, sub1.getRhsOperand().getType()) ;
- assertEquals(DataType.LONG, gt1.getLhsOperand().getType()) ;
}
@@ -790,7 +787,7 @@
// Positive case with cast insertion
@Test
- public void testRegexTypeChecking2() throws Throwable {
+ public void testRegexTypeChecking2Fail() throws Throwable {
LogicalPlan plan = new LogicalPlan() ;
LOConst constant1 = new LOConst(plan, genNewOperatorKey(), new DataByteArray()) ;
constant1.setType(DataType.BYTEARRAY) ;
@@ -805,22 +802,21 @@
CompilationMessageCollector collector = new CompilationMessageCollector() ;
TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
- typeValidator.validate(plan, collector) ;
+ try {
+ typeValidator.validate(plan, collector) ;
+ } catch (PlanValidationException pve) {
+ // good
+ }
printMessageCollector(collector) ;
printTypeGraph(plan) ;
// After type checking
- // check type
- System.out.println(DataType.findTypeName(regex.getType())) ;
- assertEquals(DataType.BOOLEAN, regex.getType()) ;
-
- // check wiring
- LOCast cast = (LOCast) regex.getOperand() ;
- assertEquals(cast.getType(), DataType.CHARARRAY);
-
- assertEquals(cast.getExpression(), constant1) ;
+ if (!collector.hasError()) {
+ throw new Exception("Error expected during type checking") ;
+ }
+
}
// Negative case
@@ -5442,6 +5438,110 @@
}
@Test
+ public void testMapLookupLineage() throws Throwable {
+ planTester.buildPlan("a = load 'a' using BinStorage() as (field1, field2: float, field3: chararray );") ;
+ planTester.buildPlan("b = foreach a generate field1#'key1' as map1;") ;
+ LogicalPlan plan = planTester.buildPlan("c = foreach b generate map1#'key2' + 1 ;") ;
+
+ // validate
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+ TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+ typeValidator.validate(plan, collector) ;
+
+ printMessageCollector(collector) ;
+ printTypeGraph(plan) ;
+ planTester.printPlan(plan, TypeCheckingTestUtil.getCurrentMethodName());
+
+ if (collector.hasError()) {
+ throw new AssertionError("Expect no error") ;
+ }
+
+
+ LOForEach foreach = (LOForEach)plan.getLeaves().get(0);
+ LogicalPlan foreachPlan = foreach.getForEachPlans().get(0);
+
+ LogicalOperator exOp = foreachPlan.getRoots().get(0);
+
+ if(! (exOp instanceof LOProject)) exOp = foreachPlan.getRoots().get(1);
+
+ LOMapLookup map = (LOMapLookup)foreachPlan.getSuccessors(exOp).get(0);
+ LOCast cast = (LOCast)foreachPlan.getSuccessors(map).get(0);
+ assertTrue(cast.getLoadFunc().toString().startsWith("org.apache.pig.builtin.BinStorage"));
+
+ }
+
+ @Test
+ public void testMapLookupLineageNoSchema() throws Throwable {
+ planTester.buildPlan("a = load 'a' using BinStorage() ;") ;
+ planTester.buildPlan("b = foreach a generate $0#'key1';") ;
+ LogicalPlan plan = planTester.buildPlan("c = foreach b generate $0#'key2' + 1 ;") ;
+
+ // validate
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+ TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+ typeValidator.validate(plan, collector) ;
+
+ printMessageCollector(collector) ;
+ printTypeGraph(plan) ;
+ planTester.printPlan(plan, TypeCheckingTestUtil.getCurrentMethodName());
+
+ if (collector.hasError()) {
+ throw new AssertionError("Expect no error") ;
+ }
+
+
+ LOForEach foreach = (LOForEach)plan.getLeaves().get(0);
+ LogicalPlan foreachPlan = foreach.getForEachPlans().get(0);
+
+ LogicalOperator exOp = foreachPlan.getRoots().get(0);
+
+ if(! (exOp instanceof LOProject)) exOp = foreachPlan.getRoots().get(1);
+
+ LOMapLookup map = (LOMapLookup)foreachPlan.getSuccessors(exOp).get(0);
+ LOCast cast = (LOCast)foreachPlan.getSuccessors(map).get(0);
+ assertTrue(cast.getLoadFunc().toString().startsWith("org.apache.pig.builtin.BinStorage"));
+
+ }
+
+ @Test
+ public void testMapLookupLineage2() throws Throwable {
+ planTester.buildPlan("a = load 'a' as (s, m, l);") ;
+ planTester.buildPlan("b = foreach a generate s#'x' as f1, s#'y' as f2, s#'z' as f3;") ;
+ planTester.buildPlan("c = group b by f1;") ;
+ LogicalPlan plan = planTester.buildPlan("d = foreach c {fil = filter b by f2 == 1; generate flatten(group), SUM(fil.f3);};") ;
+
+ // validate
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+ TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+ typeValidator.validate(plan, collector) ;
+
+ printMessageCollector(collector) ;
+ printTypeGraph(plan) ;
+ planTester.printPlan(plan, TypeCheckingTestUtil.getCurrentMethodName());
+
+ if (collector.hasError()) {
+ throw new AssertionError("Expect no error") ;
+ }
+
+
+ LOForEach foreach = (LOForEach)plan.getLeaves().get(0);
+ LogicalPlan foreachPlan = foreach.getForEachPlans().get(1);
+
+ LogicalOperator exOp = foreachPlan.getRoots().get(0);
+
+ LOFilter filter = (LOFilter)foreachPlan.getSuccessors(exOp).get(0);
+ LogicalPlan filterPlan = filter.getComparisonPlan();
+
+ exOp = filterPlan.getRoots().get(0);
+
+ if(! (exOp instanceof LOProject)) exOp = filterPlan.getRoots().get(1);
+
+
+ LOCast cast = (LOCast)filterPlan.getSuccessors(exOp).get(0);
+ assertTrue(cast.getLoadFunc().toString().startsWith("org.apache.pig.builtin.PigStorage"));
+
+ }
+
////////////////////////// Helper //////////////////////////////////
private void checkForEachCasting(LOForEach foreach, int idx, boolean isCast, byte toType) {
LogicalPlan plan = foreach.getForEachPlans().get(idx) ;