You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2010/10/08 02:22:32 UTC
svn commit: r1005678 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/LOCast.java
src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
test/org/apache/pig/test/TestUnionOnSchema.java
Author: thejas
Date: Fri Oct 8 00:22:31 2010
New Revision: 1005678
URL: http://svn.apache.org/viewvc?rev=1005678&view=rev
Log:
PIG-1666: union onschema fails when the input relation has cast from bytearray to another type
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCast.java
pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
pig/trunk/test/org/apache/pig/test/TestUnionOnSchema.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1005678&r1=1005677&r2=1005678&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Oct 8 00:22:31 2010
@@ -209,6 +209,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1666: union onschema fails when the input relation has cast from bytearray to another type (thejas)
+
PIG-1655: code duplicated for udfs that were moved from piggybank to builtin (nrai via daijy)
PIG-1670: pig throws ExecException in stead of FrontEnd exception when the plan validation fails (nrai via daijy)
Modified: pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCast.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCast.java?rev=1005678&r1=1005677&r2=1005678&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCast.java (original)
+++ pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCast.java Fri Oct 8 00:22:31 2010
@@ -83,7 +83,7 @@ public class LOCast extends ExpressionOp
public Schema.FieldSchema getFieldSchema() throws FrontendException {
if(!mIsFieldSchemaComputed) {
if(userSpecifiedFieldSchema != null) {
- mFieldSchema = userSpecifiedFieldSchema;
+ mFieldSchema = new FieldSchema(userSpecifiedFieldSchema);
} else {
mFieldSchema = new Schema.FieldSchema(null, mType);
}
Modified: pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=1005678&r1=1005677&r2=1005678&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Fri Oct 8 00:22:31 2010
@@ -1785,8 +1785,33 @@ public class TypeCheckingVisitor extends
) {
try {
Map<String, LogicalOperator> canonicalMap = cast.getFieldSchema().getCanonicalMap();
+ // two variables to ensure that only one load func is mapped to
+ // the cast operator
+ FuncSpec prevLoadFuncSpec = null;
+ boolean prevLoadFuncSet = false;
for( Map.Entry<String, LogicalOperator> entry : canonicalMap.entrySet() ) {
FuncSpec loadFuncSpec = getLoadFuncSpec( entry.getValue(), entry.getKey() );
+ if(!prevLoadFuncSet){
+ prevLoadFuncSet = true;
+ prevLoadFuncSpec = loadFuncSpec;
+ }
+ if(loadFuncSpec == null ){
+ if(prevLoadFuncSpec != null){
+ //BUG
+ String msg = "Bug: A null and a non-null load function " +
+ " mapped to cast through lineage";
+ throw new VisitorException(msg, 2258, PigException.BUG);
+ }else{
+ continue;
+ }
+ }
+ if(!loadFuncSpec.equals(prevLoadFuncSpec)){
+ //BUG
+ String msg = "Bug:Two different load functions mapped to " +
+ "an LOCast op";
+ throw new VisitorException(msg, 2258, PigException.BUG);
+ }
+
cast.setLoadFuncSpec( loadFuncSpec );
}
} catch (FrontendException fee) {
Modified: pig/trunk/test/org/apache/pig/test/TestUnionOnSchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUnionOnSchema.java?rev=1005678&r1=1005677&r2=1005678&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUnionOnSchema.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUnionOnSchema.java Fri Oct 8 00:22:31 2010
@@ -132,6 +132,35 @@ public class TestUnionOnSchema {
}
+
+ /**
+ * Test UNION ONSCHEMA with cast from bytearray to another type
+ * @throws IOException
+ * @throws ParseException
+ */
+ @Test
+ public void testUnionOnSchemaCastOnByteArray() throws IOException, ParseException {
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ String query =
+ " l1 = load '" + INP_FILE_2NUMS + "' as (i, j);"
+ + " f1 = foreach l1 generate (int)i, (int)j;"
+ + "u = union onschema f1, l1;"
+ ;
+ Util.registerMultiLineQuery(pig, query);
+ Iterator<Tuple> it = pig.openIterator("u");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "(1,2)",
+ "(5,3)",
+ "(1,2)",
+ "(5,3)"
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedRes);
+
+ }
+
/**
* Test UNION ONSCHEMA where a common column has additional 'namespace' part
* in the column name in one of the inputs