You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/10/31 20:21:29 UTC

svn commit: r1537592 [2/2] - in /pig/branches/tez: ./ conf/ ivy/ src/ src/docs/src/documentation/content/xdocs/ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src...

Modified: pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java Thu Oct 31 19:21:28 2013
@@ -319,7 +319,13 @@ public class LogicalPlanBuilder {
         }
         sort.setAscendingCols( ascFlags );
         alias = buildOp( loc, sort, alias, inputAlias, null );
-        expandAndResetVisitor(loc, sort);
+        try {
+            (new ProjectStarExpander(sort.getPlan())).visit(sort);
+            (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
+            new SchemaResetter(sort.getPlan(), true).visit(sort);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
         return alias;
     }
 
@@ -339,7 +345,13 @@ public class LogicalPlanBuilder {
         rank.setAscendingCol(ascFlags);
 
         buildOp( loc, rank, alias, inputAlias, null );
-        expandAndResetVisitor(loc, rank);
+        try {
+            (new ProjectStarExpander(rank.getPlan())).visit(rank);
+            (new ProjStarInUdfExpander(rank.getPlan())).visit(rank);
+            new SchemaResetter(rank.getPlan(), true).visit(rank);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
 
         return alias;
     }
@@ -396,20 +408,16 @@ public class LogicalPlanBuilder {
         op.setInnerFlags( flags );
         op.setJoinPlans( joinPlans );
         alias = buildOp( loc, op, alias, inputAliases, partitioner );
-        expandAndResetVisitor(loc, op);
+        try {
+            (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
         return alias;
     }
 
-    private void expandAndResetVisitor(SourceLocation loc,
-	    LogicalRelationalOperator lrop) throws ParserValidationException {
-        try {
-	    (new ProjectStarExpander(lrop.getPlan())).visit();
-	    (new ProjStarInUdfExpander(lrop.getPlan())).visit();
-	    new SchemaResetter(lrop.getPlan(), true).visit();
-	} catch (FrontendException e) {
-	    throw new ParserValidationException(intStream, loc, e);
-	}
-    }
 
     LOCube createCubeOp() {
 	return new LOCube(plan);
@@ -426,7 +434,13 @@ public class LogicalPlanBuilder {
 	op.setExpressionPlans(expressionPlans);
 	op.setOperations(operations);
 	buildOp(loc, op, alias, inputAlias, null);
-	expandAndResetVisitor(loc, op);
+        try {
+            (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
 	try {
 	    alias = convertCubeToFGPlan(loc, op, inputAlias, operations, expressionPlans);
 	} catch (FrontendException e) {
@@ -822,7 +836,13 @@ public class LogicalPlanBuilder {
         op.setGroupType( gt );
         op.setInnerFlags( flags );
         alias = buildOp( loc, op, alias, inputAliases, partitioner );
-        expandAndResetVisitor(loc, op);
+        try {
+            (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
 
         return alias;
     }
@@ -976,7 +996,13 @@ public class LogicalPlanBuilder {
     throws ParserValidationException {
         op.setInnerPlan( innerPlan );
         alias = buildOp( loc, op, alias, inputAlias, null );
-        expandAndResetVisitor(loc, op);
+        try {
+            (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
+        } catch (FrontendException e ) {
+            throw new ParserValidationException( intStream, loc, e );
+        }
         return alias;
     }
 

Modified: pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java (original)
+++ pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java Thu Oct 31 19:21:28 2013
@@ -49,6 +49,7 @@ import org.apache.pig.newplan.logical.re
 import org.apache.pig.newplan.logical.relational.LOLoad;
 import org.apache.pig.newplan.logical.relational.LOSort;
 import org.apache.pig.newplan.logical.relational.LOSplit;
+import org.apache.pig.newplan.logical.relational.LOSplitOutput;
 import org.apache.pig.newplan.logical.relational.LOUnion;
 import org.apache.pig.newplan.logical.relational.LOStore;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
@@ -212,6 +213,13 @@ public class LineageTrimmingVisitor exte
     }
 
     @Override
+    public void visit(LOSplitOutput split) throws FrontendException {
+        if(continueTrimming)
+            processOperator(split);
+
+    }
+
+    @Override
     public void visit(LOUnion u) throws FrontendException {
         if(continueTrimming)
             processOperator(u);

Modified: pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Oct 31 19:21:28 2013
@@ -186,19 +186,6 @@ public class PreprocessorContext {
             throw rte;
         }
 
-        int exitVal;
-        try {
-            exitVal = p.waitFor();
-        } catch (InterruptedException e) {
-            RuntimeException rte = new RuntimeException("Interrupted Thread Exception while waiting for command to get over"+e.getMessage() , e);
-            throw rte;
-        }
-
-        if (exitVal != 0) {
-            RuntimeException rte = new RuntimeException("Error executing shell command: " + cmd + ". Command exit with exit code of " + exitVal );
-            throw rte;
-        }
-
         BufferedReader br = null;
         try{
             InputStreamReader isr = new InputStreamReader(p.getInputStream());
@@ -235,6 +222,19 @@ public class PreprocessorContext {
             if (br != null) try {br.close();} catch(Exception e) {}
         }
 
+        int exitVal;
+        try {
+            exitVal = p.waitFor();
+        } catch (InterruptedException e) {
+            RuntimeException rte = new RuntimeException("Interrupted Thread Exception while waiting for command to get over"+e.getMessage() , e);
+            throw rte;
+        }
+
+        if (exitVal != 0) {
+            RuntimeException rte = new RuntimeException("Error executing shell command: " + cmd + ". Command exit with exit code of " + exitVal );
+            throw rte;
+        }
+
         return streamData.trim();
     }
 

Propchange: pig/branches/tez/src/pig-default.properties
------------------------------------------------------------------------------
  Merged /pig/trunk/src/pig-default.properties:r1532966-1537587

Modified: pig/branches/tez/test/e2e/pig/tests/negative.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/tests/negative.conf?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/tests/negative.conf (original)
+++ pig/branches/tez/test/e2e/pig/tests/negative.conf Thu Oct 31 19:21:28 2013
@@ -567,7 +567,7 @@ c = foreach b generate org.apache.pig.te
 d = foreach c generate $0#'alice young';
 split d into e if $0 < 42, f if $0 >= 42;
 store e into ':OUTPATH:';\,
-                'expected_err_regex' => "Received a bytearray from the UDF. Cannot determine how to convert the bytearray",
+                'expected_err_regex' => "Received a bytearray from the UDF or Union from two different Loaders. Cannot determine how to convert the bytearray to int",
             },
         ]
         }

Modified: pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java Thu Oct 31 19:21:28 2013
@@ -486,6 +486,11 @@ public class TestQueryParser {
     
     @Test
     public void testFunction2() throws Exception {
+        shouldPass("B = foreach A generate org.apache.pig.builtin.CONCAT(b, c, 'x', 'y');");
+    }
+    
+    @Test
+    public void testFunction3() throws Exception {
         shouldPass("B = foreach A generate flatten(myudfs.Swap(name, age)), gpa;");
     }
     

Modified: pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java Thu Oct 31 19:21:28 2013
@@ -26,8 +26,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.pig.ResourceStatistics;
+import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
 import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.impl.util.Utils;
 
 public class PigStorageWithStatistics extends PigStorage {
     private String loc = null;
@@ -58,7 +58,7 @@ public class PigStorageWithStatistics ex
             FileStatus[] status = fs.globStatus(path);
             if (status != null) {
                 for (FileStatus s : status) {
-                    inputBytes += Utils.getPathLength(fs, s);
+                    inputBytes += MapRedUtil.getPathLength(fs, s);
                 }
             }
         }

Modified: pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java Thu Oct 31 19:21:28 2013
@@ -2191,6 +2191,34 @@ public class TestBuiltin {
         assertTrue(msg, res.equals(exp));
 
     }
+    
+    /**
+     * End-to-end testing of the CONCAT() builtin function for vararg parameters
+     * @throws Exception
+     */
+    @Test
+    public void testComplexMultiCONCAT() throws Exception {
+        String input = "vararg_concat_test_jira_3444.txt";
+        Util.createLocalInputFile(input, new String[]{"dummy"});
+        PigServer pigServer = new PigServer(ExecType.LOCAL);
+        pigServer.registerQuery("A = LOAD '"+input+"' as (x:chararray);");
+        
+        pigServer.registerQuery("B = foreach A generate CONCAT('a', CONCAT('b',CONCAT('c','d')));");
+        Iterator<Tuple> its = pigServer.openIterator("B");
+        Tuple t = its.next();
+        assertEquals("abcd",t.get(0));
+        
+        pigServer.registerQuery("B = foreach A generate CONCAT('a', 'b', 'c', 'd');");
+        its = pigServer.openIterator("B");
+        t = its.next();
+        assertEquals("abcd",t.get(0));
+        
+        pigServer.registerQuery("B = foreach A generate CONCAT('a', CONCAT('b','c'), 'd');");
+        its = pigServer.openIterator("B");
+        t = its.next();
+        assertEquals("abcd",t.get(0));
+        
+    }
 
     @Test
     public void testSIZE() throws Exception {

Modified: pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java Thu Oct 31 19:21:28 2013
@@ -25,17 +25,19 @@ import java.util.Iterator;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
 import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.test.utils.TestHelper;
-import org.apache.pig.tools.pigstats.JobStats;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStats.JobGraph;
 import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -397,7 +399,39 @@ public class TestFRJoin2 {
             assertEquals(3, PigStats.get().getJobGraph().size());
         }
         assertEquals(dbfrj.size(), dbshj.size());
-        assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));    
+        assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+    }
+
+    @Test
+    public void testTooBigReplicatedFile() throws Exception {
+        PigServer pigServer = new PigServer(
+                ExecType.MAPREDUCE, cluster.getProperties());
+
+        pigServer.registerQuery("A = LOAD '" + INPUT_DIR + "' as (x:int,y:int);");
+        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
+        pigServer.registerQuery("C = group B all parallel 5;");
+        pigServer.registerQuery("C = foreach C generate MAX(B.x) as x;");
+        pigServer.registerQuery("D = join A by x, B by x, C by x using 'repl';");
+        {
+            // When the replicated input sizes=(12 + 5) is bigger than
+            // pig.join.replicated.max.bytes=16, we throw exception
+            try {
+                pigServer.getPigContext().getProperties().setProperty(
+                        PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES,
+                        String.valueOf(16));
+                pigServer.openIterator("D");
+                Assert.fail();
+            } catch (FrontendException e) {
+                assertEquals("Internal error. Distributed cache could" +
+                        " not be set up for the replicated files",
+                        e.getCause().getCause().getCause().getMessage());
+            }
+
+            // If we increase the size to 17, it should work
+            pigServer.getPigContext().getProperties().setProperty(
+                        PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES,
+                        String.valueOf(17));
+            pigServer.openIterator("D");
+        }
     }
-    
 }

Modified: pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java Thu Oct 31 19:21:28 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
@@ -78,7 +79,7 @@ public class TestHBaseStorage {
     public static void setUp() throws Exception {
         // This is needed by Pig
         cluster = MiniCluster.buildCluster();
-        conf = cluster.getConfiguration();
+        conf = HBaseConfiguration.create(cluster.getConfiguration());
 
         util = new HBaseTestingUtility(conf);
         util.startMiniZKCluster();

Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java Thu Oct 31 19:21:28 2013
@@ -111,7 +111,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("store D1 into '/tmp/output1';");
             myPig.registerQuery("store D2 into '/tmp/output2';");
             
-            LogicalPlan lp = checkLogicalPlan(1, 2, 9);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 12);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 15);
 
@@ -168,7 +168,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("p4 = foreach g4 generate group, COUNT(f4.gid);");
             myPig.registerQuery("store p4 into '/tmp/output4';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 4, 17);
+            LogicalPlan lp = checkLogicalPlan(1, 4, 26);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 4, 35);
 
@@ -203,7 +203,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("e = foreach d generate group, COUNT(c), COUNT(b);");
             myPig.registerQuery("store e into '/tmp/output1';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 1, 6);
+            LogicalPlan lp = checkLogicalPlan(1, 1, 9);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 1, 13);
 
@@ -235,7 +235,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("g1 = foreach g generate group, COUNT(d), COUNT(e);");
             myPig.registerQuery("store g1 into '/tmp/output2';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 2, 11);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 16);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 23);
 
@@ -268,7 +268,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("h = foreach g generate group, COUNT(f.uid);");
             myPig.registerQuery("store h into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 11);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 17);
 
             // NOTE: old way seemingly generated a useless foreach operator. Now we have one less operator. Reason unknow.
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 19);
@@ -333,7 +333,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d2 = foreach d1 generate group, AVG(d.uid);");            
             myPig.registerQuery("store d2 into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
 
@@ -365,7 +365,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("f1 = foreach f generate group, SUM(d.c::uid);");
             myPig.registerQuery("store f1 into '/tmp/output2';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 2, 10);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 16);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 25);
 
@@ -401,7 +401,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d2 = foreach d1 generate group, MAX(d.uid) - MIN(d.uid);");
             myPig.registerQuery("store d2 into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
 
@@ -437,7 +437,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d2 = foreach d1 generate group, d.uname, MAX(d.uid) - MIN(d.uid);");
             myPig.registerQuery("store d2 into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
 
@@ -473,7 +473,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d2 = foreach d1 generate group, COUNT(d.uid);");
             myPig.registerQuery("store d2 into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
 
@@ -507,7 +507,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("H = foreach G generate group, COUNT(A1);");          
             myPig.registerQuery("store H into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 15);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 24);
 
@@ -603,7 +603,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("e = join c by gid, d by gid using 'repl';");
             myPig.registerQuery("store e into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+            LogicalPlan lp = checkLogicalPlan(2, 3, 14);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 16);
 
@@ -632,7 +632,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d = foreach e generate flatten(c), flatten(b);");
             myPig.registerQuery("store d into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 9);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 15);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 19);
 
@@ -663,7 +663,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d = foreach c generate flatten(group), flatten($1), flatten($2);");
             myPig.registerQuery("store d into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 11);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 17);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 21);
 
@@ -720,7 +720,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("f = filter e by b::uid < 1000;");
             myPig.registerQuery("store f into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 9);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 22);
 
@@ -753,7 +753,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("store g into '/tmp/output1';");
             myPig.registerQuery("store h into '/tmp/output2';");
             
-            LogicalPlan lp = checkLogicalPlan(2, 2, 10);
+            LogicalPlan lp = checkLogicalPlan(2, 2, 13);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 2, 2, 20);
 
@@ -782,7 +782,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("e = filter d by $1 > 5;");
             myPig.registerQuery("store e into '/tmp/output2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 7);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 10);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 13);
 
@@ -813,7 +813,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("g = foreach f generate group, SUM(e.$0);");
             myPig.registerQuery("store g into '/tmp/output2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 9);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 12);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 17);
 
@@ -847,7 +847,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("f2 = foreach f1 generate group, COUNT(f.$0);");
             myPig.registerQuery("store f2 into '/tmp/output2';");
             
-            LogicalPlan lp = checkLogicalPlan(1, 2, 12);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 15);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 22);
 
@@ -887,7 +887,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("f5 = foreach f4 generate group, COUNT(f3.$0);");
             myPig.registerQuery("store f5 into '/tmp/output2';");
             
-            LogicalPlan lp = checkLogicalPlan(1, 2, 18);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 21);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 32);
 
@@ -923,7 +923,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d2 = foreach d1 generate group, COUNT(d.uid);");
             myPig.registerQuery("store d2 into '/tmp/output3';");
              
-            LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 18);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
 
@@ -950,7 +950,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("c = group b by gid;");
             myPig.registerQuery("store c into '/tmp/output2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 5);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 8);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
 
@@ -979,7 +979,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("d = filter c by uid > 15;");
             myPig.registerQuery("store d into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 7);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 13);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 14);
 
@@ -1010,7 +1010,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("e = cogroup c by uid, d by uid;");
             myPig.registerQuery("store e into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+            LogicalPlan lp = checkLogicalPlan(2, 3, 14);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 19);
 
@@ -1079,7 +1079,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("g = group f by $0;");
             myPig.registerQuery("store g into '/tmp/output4';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 10);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 13);
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 20);
             MROperPlan mp = checkMRPlan(pp, 1, 2, 3);
 
@@ -1191,7 +1191,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("store b into '/tmp/output2';");
             myPig.registerQuery("store b into '/tmp/output3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 5);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 9);
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 10);
             MROperPlan mp = checkMRPlan(pp, 1, 1, 1);
 
@@ -1384,7 +1384,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("STORE c INTO 'output1';");
             myPig.registerQuery("STORE l INTO 'output2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 6);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 9);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 12);
 
@@ -1420,7 +1420,7 @@ public class TestMultiQueryCompiler {
             myPig.registerQuery("STORE c1 INTO 'output1';");
             myPig.registerQuery("STORE c2 INTO 'output2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 7);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 10);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 15);
 

Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java Thu Oct 31 19:21:28 2013
@@ -96,7 +96,7 @@ public class TestMultiQueryLocal {
             myPig.registerQuery("c = group b by gid;");
             myPig.registerQuery("store c into '" + TMP_DIR + "/Pig-TestMultiQueryLocal2';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 2, 5);
+            LogicalPlan lp = checkLogicalPlan(1, 2, 8);
 
             // XXX Physical plan has one less node in the local case
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
@@ -197,7 +197,7 @@ public class TestMultiQueryLocal {
             myPig.registerQuery("d = filter c by uid > 15;");
             myPig.registerQuery("store d into '" + TMP_DIR + "/Pig-TestMultiQueryLocal3';");
 
-            LogicalPlan lp = checkLogicalPlan(1, 3, 7);
+            LogicalPlan lp = checkLogicalPlan(1, 3, 13);
 
             PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 14);
 
@@ -258,7 +258,7 @@ public class TestMultiQueryLocal {
             myPig.registerQuery("e = cogroup c by uid, d by uid;");
             myPig.registerQuery("store e into '" + TMP_DIR + "/Pig-TestMultiQueryLocal3';");
 
-            LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+            LogicalPlan lp = checkLogicalPlan(2, 3, 14);
 
             // XXX the total number of ops is one less in the local case
             PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 19);

Modified: pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java Thu Oct 31 19:21:28 2013
@@ -225,8 +225,7 @@ public class TestOptimizeLimit {
         optimizePlan(newLogicalPlan);
         LOStore store = (LOStore)newLogicalPlan.getSinks().get(0);
         LOForEach foreach1 = (LOForEach)newLogicalPlan.getPredecessors(store).get(0);
-        LOForEach foreach2 = (LOForEach)newLogicalPlan.getPredecessors(foreach1).get(0);
-        LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach2).get(0);
+        LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach1).get(0);
         Assert.assertTrue(newLogicalPlan.getSoftLinkPredecessors(limit).get(0) instanceof LOStore);
     }
 

Modified: pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Thu Oct 31 19:21:28 2013
@@ -86,17 +86,15 @@ public class TestProjectStarRangeInUdf  
     }
 
     @Test
-    public void testProjStarExpandInForeach1Negative() throws IOException{
-        //star expansion gives 3 columns, so CONCAT(*) gives error
+    public void testProjStarExpandInForeach1Multi() throws IOException{
+        //star expansion gives 3 columns, CONCAT(*) should pass
         String query;
 
         query =
             "  l1 = load '" + INP_FILE_5FIELDS + "' as (a, b, c);"
             + "f = foreach l1 generate CONCAT(*) as ct;"
             ; 
-        Util.checkExceptionMessage(query, "f",
-                "Could not infer the matching function for " +
-                "org.apache.pig.builtin.CONCAT");
+        compileAndCompareSchema("ct : bytearray", query, "f");
     }
     
     @Test

Modified: pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java Thu Oct 31 19:21:28 2013
@@ -54,6 +54,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigTextInputFormat;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.PigImplConstants;
 import org.apache.pig.impl.io.FileLocalizer;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.ObjectSerializer;
@@ -78,6 +79,7 @@ public class TestPruneColumn {
     File tmpFile10;
     File tmpFile11;
     File tmpFile12;
+    File tmpFile13;
     File logFile;
 
     private static final String simpleEchoStreamingCommand;
@@ -179,6 +181,16 @@ public class TestPruneColumn {
         ps.println("[key1#1,key2#2,cond#1]");
         ps.println("[key1#2,key2#3,cond#1]");
         ps.close();
+        
+        tmpFile13 = File.createTempFile("prune", "txt");
+        ps = new PrintStream(new FileOutputStream(tmpFile13));
+        ps.println("3\ti");
+        ps.println("3\ti");
+        ps.println("1\ti");
+        ps.println("2\ti");
+        ps.println("2\ti");
+        ps.println("3\ti");
+        ps.close();
     }
 
     @After
@@ -1548,6 +1560,41 @@ public class TestPruneColumn {
         assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1",
                 "Columns pruned for B: $1"}));
     }
+    
+    @Test
+    public void testComplex2() throws Exception {
+        HashSet<String> optimizerRules = new HashSet<String>();
+        optimizerRules.add("PushUpFilter");
+        pigServer.getPigContext().getProperties().setProperty(
+                PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
+                ObjectSerializer.serialize(optimizerRules));
+        pigServer.registerQuery("A = load '"+ Util.generateURI(Util.encodeEscape(tmpFile13.toString()), pigServer.getPigContext()) + "' as (a:int, b:chararray);");
+        pigServer.registerQuery("B = FOREACH A generate a;");
+        pigServer.registerQuery("C = GROUP B by a;");
+        pigServer.registerQuery("D = filter C by group > 0 and group < 100;");
+        pigServer.registerQuery("E = FOREACH D {F = LIMIT B 1 ;GENERATE B.a as mya, FLATTEN(F.a) as setting;}");
+        pigServer.registerQuery("G = FOREACH E GENERATE mya, setting as setting;");
+
+        Iterator<Tuple> iter = pigServer.openIterator("G");
+
+        assertTrue(iter.hasNext());
+        Tuple t = iter.next();
+        assertEquals("({(1)},1)", t.toString());
+        
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertEquals("({(2),(2)},2)", t.toString());
+        
+        assertTrue(iter.hasNext());
+        t = iter.next();
+        assertEquals("({(3),(3),(3)},3)", t.toString());
+
+        assertFalse(iter.hasNext());
+
+        assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1"}));
+        
+        pigServer.getPigContext().getProperties().remove(PigImplConstants.PIG_OPTIMIZER_RULES_KEY);
+    }
 
     @Test
     public void testCoGroup8() throws Exception {

Modified: pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Thu Oct 31 19:21:28 2013
@@ -2679,7 +2679,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a'  using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a'  using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = foreach d generate group, field1 + 1, field4 + 2.0 ;";
@@ -2690,7 +2690,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupMapLookupLineage() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = foreach d generate group, field1#'key' + 1, field4 + 2.0 ;";
@@ -2716,26 +2716,26 @@ public class TestTypeCheckingValidatorNe
             exOp = (LogicalExpression) foreachPlan.getSinks().get(0);
             if(! (exOp instanceof ProjectExpression)) exOp = (LogicalExpression) foreachPlan.getSinks().get(1);
             CastExpression cast = (CastExpression)foreachPlan.getPredecessors(exOp).get(0);
-            checkCastLoadFunc(cast, "PigStorage('b')");
+            checkCastLoadFunc(cast, "org.apache.pig.test.PigStorageWithDifferentCaster('b')");
         }
 
         @Test
         public void testCogroupStarLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by *, b by *;"
             + "d = foreach c generate group, flatten($1), flatten($2);"
             + "e = foreach d generate group, field1 + 1, field4 + 2.0;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupStarLineageFail() throws Throwable {
 
-            String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+            String query = "a = load 'a' using PigStorage('x') as (field1, field2: float, field3: chararray );"
+            + "b = load 'b' using PigStorage('x') as (field4, field5, field6: chararray );"
             + "c = cogroup a by *, b by *;"
             + "d = foreach c generate group, flatten($1), flatten($2);"
             + "e = foreach d generate group + 1, field1 + 1, field4 + 2.0;";
@@ -2774,8 +2774,8 @@ public class TestTypeCheckingValidatorNe
 
         @Test
         public void testCogroupStarLineageNoSchemaFail() throws Throwable {
-            String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'b' using PigStorage('b');"
+            String query = "a = load 'a' using PigStorage('x');"
+            + "b = load 'b' using PigStorage('x');"
             + "c = cogroup a by *, b by *;";
 
             String exMsg= "Cogroup/Group by '*' or 'x..' " +
@@ -2788,33 +2788,33 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupMultiColumnProjectLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, a.(field1, field2), b.(field4);"
             + "e = foreach d generate group, flatten($1), flatten($2);"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupProjectStarLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate *;"
             + "f = foreach d generate group, flatten(a), flatten(b) ;"
             + "g = foreach f generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupProjectStarLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'b' using PigStorage('b');"
+            + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate *;"
             + "f = foreach d generate group, flatten(a), flatten(b) ;"
@@ -2827,7 +2827,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupProjectStarLineageMixSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'b' using PigStorage();"
+            + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster();"
             + "c = cogroup a by field1, b by $0;"
             + "d = foreach c generate *;"
             + "f = foreach d generate group, flatten(a), flatten(b) ;"
@@ -2840,20 +2840,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupLineageFail() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = foreach d generate group + 1, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, null, 0);
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupLineage2NoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = foreach d generate $0, $1 + 1, $2 + 2.0 ;";
@@ -2866,7 +2866,7 @@ public class TestTypeCheckingValidatorNe
         public void testUnionLineage() throws Throwable {
             //here the type checker will insert a cast for the union, converting the column field2 into a float
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = union a , b;"
             + "d = foreach c generate field2 + 2.0 ;";
 
@@ -2888,7 +2888,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testUnionLineageNoSchemaDiffLoadFunc() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = union a , b;"
             + "d = foreach c generate $1 + 2.0 ;";
 
@@ -2909,7 +2909,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testUnionLineageDifferentSchemaFail() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray, field7 );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray, field7 );"
             + "c = union a , b;"
             + "d = foreach c generate $3 + 2.0 ;";
 
@@ -2928,7 +2928,7 @@ public class TestTypeCheckingValidatorNe
             printMessageCollector(collector);
 
             boolean isWarningSeen = false;
-            assertTrue("message collector has message", collector.hasMessage());
+            assertTrue("message collector does not have message", collector.hasMessage());
 
             for (Message msg : collector){
                 if (msg.getMessageType() == MessageType.Warning
@@ -2937,7 +2937,7 @@ public class TestTypeCheckingValidatorNe
                 }
             }
 
-            assertTrue("Expected warning is seen", isWarningSeen);
+            assertTrue("Expected warning is not seen", isWarningSeen);
         }
 
         @Test
@@ -2953,10 +2953,10 @@ public class TestTypeCheckingValidatorNe
 
         @Test
         public void testUnionLineageMixSchemaFail() throws Throwable {
-            // different loader func spec associated with each input, so can't determine
+            // different loader caster associated with each input, so can't determine
             // which one to use on union output
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = union a , b;"
             + "d = foreach c generate $3 + 2.0 ;";
 
@@ -3014,20 +3014,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupFilterLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = filter d by field4 > 5;"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupFilterLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = filter d by $2 > 5;"
@@ -3115,20 +3115,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupSplitLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-                + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+                + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
                 + "c = cogroup a by field1, b by field4;"
                 + "d = foreach c generate group, flatten(a), flatten(b) ;"
                 + "split d into e if field4 > 'm', f if field6 > 'm' ;"
                 + "g = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupSplitLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-                + "b = load 'a' using PigStorage('b');"
+                + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
                 + "c = cogroup a by $0, b by $0;"
                 + "d = foreach c generate group, flatten(a), flatten(b) ;"
                 + "split d into e if $1 > 'm', f if $1 > 'm' ;"
@@ -3158,20 +3158,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupDistinctLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = distinct d;"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupDistinctLineageNoSchema() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage();"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = distinct d;"
@@ -3231,20 +3231,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupSortLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by field4 desc;"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupSortLineageNoSchema() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by $2 desc;"
@@ -3256,20 +3256,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupSortStarLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by * desc;"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupSortStarLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by * desc;"
@@ -3282,12 +3282,12 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCrossLineage() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cross a, b;"
             + "d = foreach c generate field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')",0);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')",1);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')",1);
         }
 
         @Test
@@ -3303,7 +3303,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCrossLineageNoSchemaFail() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cross a , b;"
             + "d = foreach c generate $1 + 2.0 ;";
 
@@ -3324,7 +3324,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCrossLineageMixSchemaFail() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cross a , b;"
             + "d = foreach c generate $3 + 2.0 ;";
 
@@ -3334,12 +3334,12 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testJoinLineage() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = join a by field1, b by field4;"
             + "d = foreach c generate field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 0);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 1);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 1);
 
         }
 
@@ -3358,7 +3358,7 @@ public class TestTypeCheckingValidatorNe
             //this test case should change when we decide on what flattening a tuple or bag
             //with null schema results in a foreach flatten and hence a join
             String query =  "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage();"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
             + "c = join a by $0, b by $0;"
             + "d = foreach c generate $1 + 2.0 ;";
 
@@ -3379,7 +3379,7 @@ public class TestTypeCheckingValidatorNe
             //this test case should change when we decide on what flattening a tuple or bag
             //with null schema results in a foreach flatten and hence a join
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage();"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
             + "c = join a by field1, b by $0;"
             + "d = foreach c generate $3 + 2.0 ;";
 
@@ -3407,20 +3407,20 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupLimitLineage() throws Throwable {
             String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = limit d 100;"
             + "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupLimitLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = limit d 100;"
@@ -3433,7 +3433,7 @@ public class TestTypeCheckingValidatorNe
         @Test
         public void testCogroupTopKLineage() throws Throwable {
             String query =  "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
-            + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
             + "c = cogroup a by field1, b by field4;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by field1 desc;"
@@ -3441,13 +3441,13 @@ public class TestTypeCheckingValidatorNe
             + "g = foreach f generate group, field1 + 1, field4 + 2.0 ;";
 
             checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
-            checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+            checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
         }
 
         @Test
         public void testCogroupTopKLineageNoSchema() throws Throwable {
             String query = "a = load 'a' using PigStorage('a');"
-            + "b = load 'a' using PigStorage('b');"
+            + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
             + "c = cogroup a by $0, b by $0;"
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = order d by $2 desc;"
@@ -3496,8 +3496,13 @@ public class TestTypeCheckingValidatorNe
             + "d = foreach c generate group, flatten(a), flatten(b) ;"
             + "e = foreach d generate $0, $1 + 1, $2 + 2.0 ;";
 
-            checkLastForeachCastLoadFunc(query, null, 1);
-            checkLastForeachCastLoadFunc(query, null, 2);
+            // PigStorage and PigStreaming both returns Utf8StorageConverter so
+            // it may return either PigStorage or PigStreaming.
+            // Here, our code happens to return the first one thus
+            // checking with PigStorage.  Depending on the implementation,
+            // this test may start failing.
+            checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
+            checkLastForeachCastLoadFunc(query, "PigStorage('a')", 2);
         }
 
         @Test

Modified: pig/branches/tez/test/org/apache/pig/test/TestUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestUDF.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestUDF.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestUDF.java Thu Oct 31 19:21:28 2013
@@ -211,6 +211,21 @@ public class TestUDF {
         pig.dumpSchema("c");
         pig.dumpSchema("d");
     }
+    
+    @Test
+    public void testEvalFuncGetVarArgToFunc() throws Exception {
+        String input = "udf_test_jira_3444.txt";
+        Util.createLocalInputFile(input, new String[]{"dummy"});
+        PigServer pigServer = new PigServer(ExecType.LOCAL);
+        pigServer.registerQuery("A = LOAD '"+input+"' USING PigStorage(',') AS (x:chararray);");
+        pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithVarArgs(3);");
+        pigServer.registerQuery("C = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithVarArgs(1,2,3,4);");
+
+        Iterator<Tuple> it = pigServer.openIterator("B");
+        assertEquals(Integer.valueOf(3),(Integer)it.next().get(0));
+        it = pigServer.openIterator("C");
+        assertEquals(Integer.valueOf(10), (Integer)it.next().get(0));
+    }
 
     @Test(expected = FrontendException.class)
     public void testEnsureProperSchema2() throws Exception {
@@ -272,4 +287,33 @@ public class TestUDF {
             return l;
         }
     }
+    
+    public static class UdfWithFuncSpecWithVarArgs extends EvalFunc<Integer> {
+        public UdfWithFuncSpecWithVarArgs() {}
+
+        @Override
+        public Integer exec(Tuple input) throws IOException {
+            int res = 0;
+            if (input == null || input.size() == 0) {
+                return res;
+            }
+            for (int i = 0; i < input.size(); i++) {
+                res += (Integer)input.get(i);
+            }
+            return res;
+        }
+        
+        @Override
+        public SchemaType getSchemaType() {
+            return SchemaType.VARARG;
+        }
+
+        @Override
+        public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+            List<FuncSpec> l = new ArrayList<FuncSpec>();
+            Schema s1 = new Schema(new Schema.FieldSchema(null,DataType.INTEGER));
+            l.add(new FuncSpec(this.getClass().getName(), s1));
+            return l;
+        }
+    }
 }
\ No newline at end of file

Modified: pig/branches/tez/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestUnion.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestUnion.java Thu Oct 31 19:21:28 2013
@@ -278,4 +278,94 @@ public class TestUnion {
         assertEquals(2, recordCount);
 
     }
-}
\ No newline at end of file
+    @Test
+    public void testCastingAfterUnionWithMultipleLoadersDifferentCasters()
+        throws Exception {
+        // Note that different caster case only works when each field is still coming
+        // from the single Loader.
+        // In the case below, 'a' is coming from A(PigStorage)
+        // and 'b' is coming from B(TextLoader). No overlaps.
+        File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"1","2","3"});
+        File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"a","b","c"});
+
+        PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+        //PigStorage and TextLoader have different LoadCasters
+        ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);");
+        ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (b:bytearray);");
+        ps.registerQuery("C = union onschema A,B;");
+        ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
+
+        Schema dumpSchema = ps.dumpSchema("D");
+        Schema expected = new Schema ();
+        expected.add(new Schema.FieldSchema("a", DataType.INTEGER));
+        expected.add(new Schema.FieldSchema("b", DataType.CHARARRAY));
+        assertEquals(expected, dumpSchema);
+
+        Iterator<Tuple> itr = ps.openIterator("D");
+        int recordCount = 0;
+        while(itr.next() != null)
+            ++recordCount;
+        assertEquals(6, recordCount);
+
+    }
+
+    @Test
+    public void testCastingAfterUnionWithMultipleLoadersDifferentCasters2()
+        throws Exception {
+        // A bit more complicated pattern but still same requirement of each
+        // field coming from the same Loader.
+        // 'a' is coming from A(PigStorage)
+        // 'i' is coming from B and C but both from the TextLoader.
+        File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"b","c", "1", "3"});
+        File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"a","b","c"});
+        File f3 = Util.createInputFile("tmp", "i3.txt", new String[] {"1","2","3"});
+
+        PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+        ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);"); // Using PigStorage()
+        ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
+        ps.registerQuery("C = load '" + f3.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
+        ps.registerQuery("B2 = join B by i, A by a;");              //{A::a: bytearray,B::i: bytearray}
+        ps.registerQuery("B3 = foreach B2 generate a, B::i as i;"); //{A::a: bytearray,i: bytearray}
+        ps.registerQuery("C2 = join C by i, A by a;");              //{A::a: bytearray,C::i: bytearray}
+        ps.registerQuery("C3 = foreach C2 generate a, C::i as i;"); //{A::a: bytearray,i: bytearray}
+        ps.registerQuery("D = union onschema B3,C3;");              // {A::a: bytearray,i: bytearray}
+        ps.registerQuery("E = foreach D generate (chararray) a, (chararray) i;");//{A::a: chararray,i: chararray}
+        Iterator<Tuple> itr = ps.openIterator("E");
+        int recordCount = 0;
+        while(itr.next() != null)
+            ++recordCount;
+        assertEquals(4, recordCount);
+
+    }
+
+    @Test
+    public void testCastingAfterUnionWithMultipleLoadersSameCaster()
+        throws Exception {
+        // Fields coming from different loaders but
+        // having the same LoadCaster.
+        File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"1\ta","2\tb","3\tc"});
+        PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+        // PigStorage and PigStorageWithStatistics have the same
+        // LoadCaster(== Utf8StorageConverter)
+        ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray, b:bytearray);");
+        ps.registerQuery("B = load '" + f1.getAbsolutePath() +
+          "' using org.apache.pig.test.PigStorageWithStatistics() as (a:bytearray, b:bytearray);");
+        ps.registerQuery("C = union onschema A,B;");
+        ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
+        // 'a' is coming from A and 'b' is coming from B; No overlaps.
+
+        Schema dumpSchema = ps.dumpSchema("D");
+        Schema expected = new Schema ();
+        expected.add(new Schema.FieldSchema("a", DataType.INTEGER));
+        expected.add(new Schema.FieldSchema("b", DataType.CHARARRAY));
+        assertEquals(expected, dumpSchema);
+
+        Iterator<Tuple> itr = ps.openIterator("D");
+        int recordCount = 0;
+        while(itr.next() != null)
+            ++recordCount;
+        assertEquals(6, recordCount);
+
+    }
+
+}

Propchange: pig/branches/tez/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2
------------------------------------------------------------------------------
  Merged /pig/trunk/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2:r1532966-1537587