You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/10/31 20:21:29 UTC
svn commit: r1537592 [2/2] - in /pig/branches/tez: ./ conf/ ivy/ src/
src/docs/src/documentation/content/xdocs/ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src...
Modified: pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java Thu Oct 31 19:21:28 2013
@@ -319,7 +319,13 @@ public class LogicalPlanBuilder {
}
sort.setAscendingCols( ascFlags );
alias = buildOp( loc, sort, alias, inputAlias, null );
- expandAndResetVisitor(loc, sort);
+ try {
+ (new ProjectStarExpander(sort.getPlan())).visit(sort);
+ (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
+ new SchemaResetter(sort.getPlan(), true).visit(sort);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
return alias;
}
@@ -339,7 +345,13 @@ public class LogicalPlanBuilder {
rank.setAscendingCol(ascFlags);
buildOp( loc, rank, alias, inputAlias, null );
- expandAndResetVisitor(loc, rank);
+ try {
+ (new ProjectStarExpander(rank.getPlan())).visit(rank);
+ (new ProjStarInUdfExpander(rank.getPlan())).visit(rank);
+ new SchemaResetter(rank.getPlan(), true).visit(rank);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
return alias;
}
@@ -396,20 +408,16 @@ public class LogicalPlanBuilder {
op.setInnerFlags( flags );
op.setJoinPlans( joinPlans );
alias = buildOp( loc, op, alias, inputAliases, partitioner );
- expandAndResetVisitor(loc, op);
+ try {
+ (new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
return alias;
}
- private void expandAndResetVisitor(SourceLocation loc,
- LogicalRelationalOperator lrop) throws ParserValidationException {
- try {
- (new ProjectStarExpander(lrop.getPlan())).visit();
- (new ProjStarInUdfExpander(lrop.getPlan())).visit();
- new SchemaResetter(lrop.getPlan(), true).visit();
- } catch (FrontendException e) {
- throw new ParserValidationException(intStream, loc, e);
- }
- }
LOCube createCubeOp() {
return new LOCube(plan);
@@ -426,7 +434,13 @@ public class LogicalPlanBuilder {
op.setExpressionPlans(expressionPlans);
op.setOperations(operations);
buildOp(loc, op, alias, inputAlias, null);
- expandAndResetVisitor(loc, op);
+ try {
+ (new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
try {
alias = convertCubeToFGPlan(loc, op, inputAlias, operations, expressionPlans);
} catch (FrontendException e) {
@@ -822,7 +836,13 @@ public class LogicalPlanBuilder {
op.setGroupType( gt );
op.setInnerFlags( flags );
alias = buildOp( loc, op, alias, inputAliases, partitioner );
- expandAndResetVisitor(loc, op);
+ try {
+ (new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
return alias;
}
@@ -976,7 +996,13 @@ public class LogicalPlanBuilder {
throws ParserValidationException {
op.setInnerPlan( innerPlan );
alias = buildOp( loc, op, alias, inputAlias, null );
- expandAndResetVisitor(loc, op);
+ try {
+ (new ProjectStarExpander(op.getPlan())).visit(op);
+ (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+ new SchemaResetter(op.getPlan(), true).visit(op);
+ } catch (FrontendException e ) {
+ throw new ParserValidationException( intStream, loc, e );
+ }
return alias;
}
Modified: pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java (original)
+++ pig/branches/tez/src/org/apache/pig/pen/LineageTrimmingVisitor.java Thu Oct 31 19:21:28 2013
@@ -49,6 +49,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LOSplit;
+import org.apache.pig.newplan.logical.relational.LOSplitOutput;
import org.apache.pig.newplan.logical.relational.LOUnion;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
@@ -212,6 +213,13 @@ public class LineageTrimmingVisitor exte
}
@Override
+ public void visit(LOSplitOutput split) throws FrontendException {
+ if(continueTrimming)
+ processOperator(split);
+
+ }
+
+ @Override
public void visit(LOUnion u) throws FrontendException {
if(continueTrimming)
processOperator(u);
Modified: pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Oct 31 19:21:28 2013
@@ -186,19 +186,6 @@ public class PreprocessorContext {
throw rte;
}
- int exitVal;
- try {
- exitVal = p.waitFor();
- } catch (InterruptedException e) {
- RuntimeException rte = new RuntimeException("Interrupted Thread Exception while waiting for command to get over"+e.getMessage() , e);
- throw rte;
- }
-
- if (exitVal != 0) {
- RuntimeException rte = new RuntimeException("Error executing shell command: " + cmd + ". Command exit with exit code of " + exitVal );
- throw rte;
- }
-
BufferedReader br = null;
try{
InputStreamReader isr = new InputStreamReader(p.getInputStream());
@@ -235,6 +222,19 @@ public class PreprocessorContext {
if (br != null) try {br.close();} catch(Exception e) {}
}
+ int exitVal;
+ try {
+ exitVal = p.waitFor();
+ } catch (InterruptedException e) {
+ RuntimeException rte = new RuntimeException("Interrupted Thread Exception while waiting for command to get over"+e.getMessage() , e);
+ throw rte;
+ }
+
+ if (exitVal != 0) {
+ RuntimeException rte = new RuntimeException("Error executing shell command: " + cmd + ". Command exit with exit code of " + exitVal );
+ throw rte;
+ }
+
return streamData.trim();
}
Propchange: pig/branches/tez/src/pig-default.properties
------------------------------------------------------------------------------
Merged /pig/trunk/src/pig-default.properties:r1532966-1537587
Modified: pig/branches/tez/test/e2e/pig/tests/negative.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/tests/negative.conf?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/tests/negative.conf (original)
+++ pig/branches/tez/test/e2e/pig/tests/negative.conf Thu Oct 31 19:21:28 2013
@@ -567,7 +567,7 @@ c = foreach b generate org.apache.pig.te
d = foreach c generate $0#'alice young';
split d into e if $0 < 42, f if $0 >= 42;
store e into ':OUTPATH:';\,
- 'expected_err_regex' => "Received a bytearray from the UDF. Cannot determine how to convert the bytearray",
+ 'expected_err_regex' => "Received a bytearray from the UDF or Union from two different Loaders. Cannot determine how to convert the bytearray to int",
},
]
}
Modified: pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/parser/TestQueryParser.java Thu Oct 31 19:21:28 2013
@@ -486,6 +486,11 @@ public class TestQueryParser {
@Test
public void testFunction2() throws Exception {
+ shouldPass("B = foreach A generate org.apache.pig.builtin.CONCAT(b, c, 'x', 'y');");
+ }
+
+ @Test
+ public void testFunction3() throws Exception {
shouldPass("B = foreach A generate flatten(myudfs.Swap(name, age)), gpa;");
}
Modified: pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/PigStorageWithStatistics.java Thu Oct 31 19:21:28 2013
@@ -26,8 +26,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ResourceStatistics;
+import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.impl.util.Utils;
public class PigStorageWithStatistics extends PigStorage {
private String loc = null;
@@ -58,7 +58,7 @@ public class PigStorageWithStatistics ex
FileStatus[] status = fs.globStatus(path);
if (status != null) {
for (FileStatus s : status) {
- inputBytes += Utils.getPathLength(fs, s);
+ inputBytes += MapRedUtil.getPathLength(fs, s);
}
}
}
Modified: pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestBuiltin.java Thu Oct 31 19:21:28 2013
@@ -2191,6 +2191,34 @@ public class TestBuiltin {
assertTrue(msg, res.equals(exp));
}
+
+ /**
+ * End-to-end testing of the CONCAT() builtin function for vararg parameters
+ * @throws Exception
+ */
+ @Test
+ public void testComplexMultiCONCAT() throws Exception {
+ String input = "vararg_concat_test_jira_3444.txt";
+ Util.createLocalInputFile(input, new String[]{"dummy"});
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ pigServer.registerQuery("A = LOAD '"+input+"' as (x:chararray);");
+
+ pigServer.registerQuery("B = foreach A generate CONCAT('a', CONCAT('b',CONCAT('c','d')));");
+ Iterator<Tuple> its = pigServer.openIterator("B");
+ Tuple t = its.next();
+ assertEquals("abcd",t.get(0));
+
+ pigServer.registerQuery("B = foreach A generate CONCAT('a', 'b', 'c', 'd');");
+ its = pigServer.openIterator("B");
+ t = its.next();
+ assertEquals("abcd",t.get(0));
+
+ pigServer.registerQuery("B = foreach A generate CONCAT('a', CONCAT('b','c'), 'd');");
+ its = pigServer.openIterator("B");
+ t = its.next();
+ assertEquals("abcd",t.get(0));
+
+ }
@Test
public void testSIZE() throws Exception {
Modified: pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestFRJoin2.java Thu Oct 31 19:21:28 2013
@@ -25,17 +25,19 @@ import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.test.utils.TestHelper;
-import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -397,7 +399,39 @@ public class TestFRJoin2 {
assertEquals(3, PigStats.get().getJobGraph().size());
}
assertEquals(dbfrj.size(), dbshj.size());
- assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+ assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+ }
+
+ @Test
+ public void testTooBigReplicatedFile() throws Exception {
+ PigServer pigServer = new PigServer(
+ ExecType.MAPREDUCE, cluster.getProperties());
+
+ pigServer.registerQuery("A = LOAD '" + INPUT_DIR + "' as (x:int,y:int);");
+ pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
+ pigServer.registerQuery("C = group B all parallel 5;");
+ pigServer.registerQuery("C = foreach C generate MAX(B.x) as x;");
+ pigServer.registerQuery("D = join A by x, B by x, C by x using 'repl';");
+ {
+ // When the replicated input sizes=(12 + 5) is bigger than
+ // pig.join.replicated.max.bytes=16, we throw exception
+ try {
+ pigServer.getPigContext().getProperties().setProperty(
+ PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES,
+ String.valueOf(16));
+ pigServer.openIterator("D");
+ Assert.fail();
+ } catch (FrontendException e) {
+ assertEquals("Internal error. Distributed cache could" +
+ " not be set up for the replicated files",
+ e.getCause().getCause().getCause().getMessage());
+ }
+
+ // If we increase the size to 17, it should work
+ pigServer.getPigContext().getProperties().setProperty(
+ PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES,
+ String.valueOf(17));
+ pigServer.openIterator("D");
+ }
}
-
}
Modified: pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestHBaseStorage.java Thu Oct 31 19:21:28 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
@@ -78,7 +79,7 @@ public class TestHBaseStorage {
public static void setUp() throws Exception {
// This is needed by Pig
cluster = MiniCluster.buildCluster();
- conf = cluster.getConfiguration();
+ conf = HBaseConfiguration.create(cluster.getConfiguration());
util = new HBaseTestingUtility(conf);
util.startMiniZKCluster();
Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java Thu Oct 31 19:21:28 2013
@@ -111,7 +111,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("store D1 into '/tmp/output1';");
myPig.registerQuery("store D2 into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 9);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 12);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 15);
@@ -168,7 +168,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("p4 = foreach g4 generate group, COUNT(f4.gid);");
myPig.registerQuery("store p4 into '/tmp/output4';");
- LogicalPlan lp = checkLogicalPlan(1, 4, 17);
+ LogicalPlan lp = checkLogicalPlan(1, 4, 26);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 4, 35);
@@ -203,7 +203,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("e = foreach d generate group, COUNT(c), COUNT(b);");
myPig.registerQuery("store e into '/tmp/output1';");
- LogicalPlan lp = checkLogicalPlan(1, 1, 6);
+ LogicalPlan lp = checkLogicalPlan(1, 1, 9);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 1, 13);
@@ -235,7 +235,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("g1 = foreach g generate group, COUNT(d), COUNT(e);");
myPig.registerQuery("store g1 into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 11);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 16);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 23);
@@ -268,7 +268,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("h = foreach g generate group, COUNT(f.uid);");
myPig.registerQuery("store h into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 11);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 17);
// NOTE: old way seemingly generated a useless foreach operator. Now we have one less operator. Reason unknow.
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 19);
@@ -333,7 +333,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d2 = foreach d1 generate group, AVG(d.uid);");
myPig.registerQuery("store d2 into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
@@ -365,7 +365,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("f1 = foreach f generate group, SUM(d.c::uid);");
myPig.registerQuery("store f1 into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 10);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 16);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 25);
@@ -401,7 +401,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d2 = foreach d1 generate group, MAX(d.uid) - MIN(d.uid);");
myPig.registerQuery("store d2 into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
@@ -437,7 +437,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d2 = foreach d1 generate group, d.uname, MAX(d.uid) - MIN(d.uid);");
myPig.registerQuery("store d2 into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
@@ -473,7 +473,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d2 = foreach d1 generate group, COUNT(d.uid);");
myPig.registerQuery("store d2 into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
@@ -507,7 +507,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("H = foreach G generate group, COUNT(A1);");
myPig.registerQuery("store H into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 15);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 24);
@@ -603,7 +603,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("e = join c by gid, d by gid using 'repl';");
myPig.registerQuery("store e into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+ LogicalPlan lp = checkLogicalPlan(2, 3, 14);
PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 16);
@@ -632,7 +632,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d = foreach e generate flatten(c), flatten(b);");
myPig.registerQuery("store d into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 9);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 15);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 19);
@@ -663,7 +663,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d = foreach c generate flatten(group), flatten($1), flatten($2);");
myPig.registerQuery("store d into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 11);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 17);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 21);
@@ -720,7 +720,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("f = filter e by b::uid < 1000;");
myPig.registerQuery("store f into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 9);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 22);
@@ -753,7 +753,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("store g into '/tmp/output1';");
myPig.registerQuery("store h into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(2, 2, 10);
+ LogicalPlan lp = checkLogicalPlan(2, 2, 13);
PhysicalPlan pp = checkPhysicalPlan(lp, 2, 2, 20);
@@ -782,7 +782,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("e = filter d by $1 > 5;");
myPig.registerQuery("store e into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 7);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 10);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 13);
@@ -813,7 +813,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("g = foreach f generate group, SUM(e.$0);");
myPig.registerQuery("store g into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 9);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 12);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 17);
@@ -847,7 +847,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("f2 = foreach f1 generate group, COUNT(f.$0);");
myPig.registerQuery("store f2 into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 12);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 15);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 22);
@@ -887,7 +887,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("f5 = foreach f4 generate group, COUNT(f3.$0);");
myPig.registerQuery("store f5 into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 18);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 21);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 32);
@@ -923,7 +923,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d2 = foreach d1 generate group, COUNT(d.uid);");
myPig.registerQuery("store d2 into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 14);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 18);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 25);
@@ -950,7 +950,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("c = group b by gid;");
myPig.registerQuery("store c into '/tmp/output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 5);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 8);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
@@ -979,7 +979,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("d = filter c by uid > 15;");
myPig.registerQuery("store d into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 7);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 13);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 14);
@@ -1010,7 +1010,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("e = cogroup c by uid, d by uid;");
myPig.registerQuery("store e into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+ LogicalPlan lp = checkLogicalPlan(2, 3, 14);
PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 19);
@@ -1079,7 +1079,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("g = group f by $0;");
myPig.registerQuery("store g into '/tmp/output4';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 10);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 13);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 20);
MROperPlan mp = checkMRPlan(pp, 1, 2, 3);
@@ -1191,7 +1191,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("store b into '/tmp/output2';");
myPig.registerQuery("store b into '/tmp/output3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 5);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 9);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 10);
MROperPlan mp = checkMRPlan(pp, 1, 1, 1);
@@ -1384,7 +1384,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("STORE c INTO 'output1';");
myPig.registerQuery("STORE l INTO 'output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 6);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 9);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 12);
@@ -1420,7 +1420,7 @@ public class TestMultiQueryCompiler {
myPig.registerQuery("STORE c1 INTO 'output1';");
myPig.registerQuery("STORE c2 INTO 'output2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 7);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 10);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 15);
Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java Thu Oct 31 19:21:28 2013
@@ -96,7 +96,7 @@ public class TestMultiQueryLocal {
myPig.registerQuery("c = group b by gid;");
myPig.registerQuery("store c into '" + TMP_DIR + "/Pig-TestMultiQueryLocal2';");
- LogicalPlan lp = checkLogicalPlan(1, 2, 5);
+ LogicalPlan lp = checkLogicalPlan(1, 2, 8);
// XXX Physical plan has one less node in the local case
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
@@ -197,7 +197,7 @@ public class TestMultiQueryLocal {
myPig.registerQuery("d = filter c by uid > 15;");
myPig.registerQuery("store d into '" + TMP_DIR + "/Pig-TestMultiQueryLocal3';");
- LogicalPlan lp = checkLogicalPlan(1, 3, 7);
+ LogicalPlan lp = checkLogicalPlan(1, 3, 13);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 14);
@@ -258,7 +258,7 @@ public class TestMultiQueryLocal {
myPig.registerQuery("e = cogroup c by uid, d by uid;");
myPig.registerQuery("store e into '" + TMP_DIR + "/Pig-TestMultiQueryLocal3';");
- LogicalPlan lp = checkLogicalPlan(2, 3, 8);
+ LogicalPlan lp = checkLogicalPlan(2, 3, 14);
// XXX the total number of ops is one less in the local case
PhysicalPlan pp = checkPhysicalPlan(lp, 2, 3, 19);
Modified: pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestOptimizeLimit.java Thu Oct 31 19:21:28 2013
@@ -225,8 +225,7 @@ public class TestOptimizeLimit {
optimizePlan(newLogicalPlan);
LOStore store = (LOStore)newLogicalPlan.getSinks().get(0);
LOForEach foreach1 = (LOForEach)newLogicalPlan.getPredecessors(store).get(0);
- LOForEach foreach2 = (LOForEach)newLogicalPlan.getPredecessors(foreach1).get(0);
- LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach2).get(0);
+ LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach1).get(0);
Assert.assertTrue(newLogicalPlan.getSoftLinkPredecessors(limit).get(0) instanceof LOStore);
}
Modified: pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Thu Oct 31 19:21:28 2013
@@ -86,17 +86,15 @@ public class TestProjectStarRangeInUdf
}
@Test
- public void testProjStarExpandInForeach1Negative() throws IOException{
- //star expansion gives 3 columns, so CONCAT(*) gives error
+ public void testProjStarExpandInForeach1Multi() throws IOException{
+ //star expansion gives 3 columns, CONCAT(*) should pass
String query;
query =
" l1 = load '" + INP_FILE_5FIELDS + "' as (a, b, c);"
+ "f = foreach l1 generate CONCAT(*) as ct;"
;
- Util.checkExceptionMessage(query, "f",
- "Could not infer the matching function for " +
- "org.apache.pig.builtin.CONCAT");
+ compileAndCompareSchema("ct : bytearray", query, "f");
}
@Test
Modified: pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPruneColumn.java Thu Oct 31 19:21:28 2013
@@ -54,6 +54,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigTextInputFormat;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.PigImplConstants;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.ObjectSerializer;
@@ -78,6 +79,7 @@ public class TestPruneColumn {
File tmpFile10;
File tmpFile11;
File tmpFile12;
+ File tmpFile13;
File logFile;
private static final String simpleEchoStreamingCommand;
@@ -179,6 +181,16 @@ public class TestPruneColumn {
ps.println("[key1#1,key2#2,cond#1]");
ps.println("[key1#2,key2#3,cond#1]");
ps.close();
+
+ tmpFile13 = File.createTempFile("prune", "txt");
+ ps = new PrintStream(new FileOutputStream(tmpFile13));
+ ps.println("3\ti");
+ ps.println("3\ti");
+ ps.println("1\ti");
+ ps.println("2\ti");
+ ps.println("2\ti");
+ ps.println("3\ti");
+ ps.close();
}
@After
@@ -1548,6 +1560,41 @@ public class TestPruneColumn {
assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1",
"Columns pruned for B: $1"}));
}
+
+ @Test
+ public void testComplex2() throws Exception {
+ HashSet<String> optimizerRules = new HashSet<String>();
+ optimizerRules.add("PushUpFilter");
+ pigServer.getPigContext().getProperties().setProperty(
+ PigImplConstants.PIG_OPTIMIZER_RULES_KEY,
+ ObjectSerializer.serialize(optimizerRules));
+ pigServer.registerQuery("A = load '"+ Util.generateURI(Util.encodeEscape(tmpFile13.toString()), pigServer.getPigContext()) + "' as (a:int, b:chararray);");
+ pigServer.registerQuery("B = FOREACH A generate a;");
+ pigServer.registerQuery("C = GROUP B by a;");
+ pigServer.registerQuery("D = filter C by group > 0 and group < 100;");
+ pigServer.registerQuery("E = FOREACH D {F = LIMIT B 1 ;GENERATE B.a as mya, FLATTEN(F.a) as setting;}");
+ pigServer.registerQuery("G = FOREACH E GENERATE mya, setting as setting;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("G");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertEquals("({(1)},1)", t.toString());
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertEquals("({(2),(2)},2)", t.toString());
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertEquals("({(3),(3),(3)},3)", t.toString());
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1"}));
+
+ pigServer.getPigContext().getProperties().remove(PigImplConstants.PIG_OPTIMIZER_RULES_KEY);
+ }
@Test
public void testCoGroup8() throws Exception {
Modified: pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Thu Oct 31 19:21:28 2013
@@ -2679,7 +2679,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = foreach d generate group, field1 + 1, field4 + 2.0 ;";
@@ -2690,7 +2690,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupMapLookupLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = foreach d generate group, field1#'key' + 1, field4 + 2.0 ;";
@@ -2716,26 +2716,26 @@ public class TestTypeCheckingValidatorNe
exOp = (LogicalExpression) foreachPlan.getSinks().get(0);
if(! (exOp instanceof ProjectExpression)) exOp = (LogicalExpression) foreachPlan.getSinks().get(1);
CastExpression cast = (CastExpression)foreachPlan.getPredecessors(exOp).get(0);
- checkCastLoadFunc(cast, "PigStorage('b')");
+ checkCastLoadFunc(cast, "org.apache.pig.test.PigStorageWithDifferentCaster('b')");
}
@Test
public void testCogroupStarLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by *, b by *;"
+ "d = foreach c generate group, flatten($1), flatten($2);"
+ "e = foreach d generate group, field1 + 1, field4 + 2.0;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupStarLineageFail() throws Throwable {
- String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+ String query = "a = load 'a' using PigStorage('x') as (field1, field2: float, field3: chararray );"
+ + "b = load 'b' using PigStorage('x') as (field4, field5, field6: chararray );"
+ "c = cogroup a by *, b by *;"
+ "d = foreach c generate group, flatten($1), flatten($2);"
+ "e = foreach d generate group + 1, field1 + 1, field4 + 2.0;";
@@ -2774,8 +2774,8 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupStarLineageNoSchemaFail() throws Throwable {
- String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'b' using PigStorage('b');"
+ String query = "a = load 'a' using PigStorage('x');"
+ + "b = load 'b' using PigStorage('x');"
+ "c = cogroup a by *, b by *;";
String exMsg= "Cogroup/Group by '*' or 'x..' " +
@@ -2788,33 +2788,33 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupMultiColumnProjectLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, a.(field1, field2), b.(field4);"
+ "e = foreach d generate group, flatten($1), flatten($2);"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupProjectStarLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'b' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate *;"
+ "f = foreach d generate group, flatten(a), flatten(b) ;"
+ "g = foreach f generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupProjectStarLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'b' using PigStorage('b');"
+ + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate *;"
+ "f = foreach d generate group, flatten(a), flatten(b) ;"
@@ -2827,7 +2827,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupProjectStarLineageMixSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'b' using PigStorage();"
+ + "b = load 'b' using org.apache.pig.test.PigStorageWithDifferentCaster();"
+ "c = cogroup a by field1, b by $0;"
+ "d = foreach c generate *;"
+ "f = foreach d generate group, flatten(a), flatten(b) ;"
@@ -2840,20 +2840,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupLineageFail() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = foreach d generate group + 1, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, null, 0);
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupLineage2NoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = foreach d generate $0, $1 + 1, $2 + 2.0 ;";
@@ -2866,7 +2866,7 @@ public class TestTypeCheckingValidatorNe
public void testUnionLineage() throws Throwable {
//here the type checker will insert a cast for the union, converting the column field2 into a float
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = union a , b;"
+ "d = foreach c generate field2 + 2.0 ;";
@@ -2888,7 +2888,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testUnionLineageNoSchemaDiffLoadFunc() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = union a , b;"
+ "d = foreach c generate $1 + 2.0 ;";
@@ -2909,7 +2909,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testUnionLineageDifferentSchemaFail() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray, field7 );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray, field7 );"
+ "c = union a , b;"
+ "d = foreach c generate $3 + 2.0 ;";
@@ -2928,7 +2928,7 @@ public class TestTypeCheckingValidatorNe
printMessageCollector(collector);
boolean isWarningSeen = false;
- assertTrue("message collector has message", collector.hasMessage());
+ assertTrue("message collector does not have message", collector.hasMessage());
for (Message msg : collector){
if (msg.getMessageType() == MessageType.Warning
@@ -2937,7 +2937,7 @@ public class TestTypeCheckingValidatorNe
}
}
- assertTrue("Expected warning is seen", isWarningSeen);
+ assertTrue("Expected warning is not seen", isWarningSeen);
}
@Test
@@ -2953,10 +2953,10 @@ public class TestTypeCheckingValidatorNe
@Test
public void testUnionLineageMixSchemaFail() throws Throwable {
- // different loader func spec associated with each input, so can't determine
+ // different loader caster associated with each input, so can't determine
// which one to use on union output
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = union a , b;"
+ "d = foreach c generate $3 + 2.0 ;";
@@ -3014,20 +3014,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupFilterLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = filter d by field4 > 5;"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupFilterLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = filter d by $2 > 5;"
@@ -3115,20 +3115,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupSplitLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "split d into e if field4 > 'm', f if field6 > 'm' ;"
+ "g = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupSplitLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "split d into e if $1 > 'm', f if $1 > 'm' ;"
@@ -3158,20 +3158,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupDistinctLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = distinct d;"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupDistinctLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage();"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = distinct d;"
@@ -3231,20 +3231,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupSortLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by field4 desc;"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupSortLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by $2 desc;"
@@ -3256,20 +3256,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupSortStarLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by * desc;"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupSortStarLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by * desc;"
@@ -3282,12 +3282,12 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCrossLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cross a, b;"
+ "d = foreach c generate field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')",0);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')",1);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')",1);
}
@Test
@@ -3303,7 +3303,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCrossLineageNoSchemaFail() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cross a , b;"
+ "d = foreach c generate $1 + 2.0 ;";
@@ -3324,7 +3324,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCrossLineageMixSchemaFail() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cross a , b;"
+ "d = foreach c generate $3 + 2.0 ;";
@@ -3334,12 +3334,12 @@ public class TestTypeCheckingValidatorNe
@Test
public void testJoinLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = join a by field1, b by field4;"
+ "d = foreach c generate field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 0);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 1);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 1);
}
@@ -3358,7 +3358,7 @@ public class TestTypeCheckingValidatorNe
//this test case should change when we decide on what flattening a tuple or bag
//with null schema results in a foreach flatten and hence a join
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage();"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
+ "c = join a by $0, b by $0;"
+ "d = foreach c generate $1 + 2.0 ;";
@@ -3379,7 +3379,7 @@ public class TestTypeCheckingValidatorNe
//this test case should change when we decide on what flattening a tuple or bag
//with null schema results in a foreach flatten and hence a join
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage();"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster();"
+ "c = join a by field1, b by $0;"
+ "d = foreach c generate $3 + 2.0 ;";
@@ -3407,20 +3407,20 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupLimitLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = limit d 100;"
+ "f = foreach e generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupLimitLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = limit d 100;"
@@ -3433,7 +3433,7 @@ public class TestTypeCheckingValidatorNe
@Test
public void testCogroupTopKLineage() throws Throwable {
String query = "a = load 'a' using PigStorage('a') as (field1, field2: float, field3: chararray );"
- + "b = load 'a' using PigStorage('b') as (field4, field5, field6: chararray );"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b') as (field4, field5, field6: chararray );"
+ "c = cogroup a by field1, b by field4;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by field1 desc;"
@@ -3441,13 +3441,13 @@ public class TestTypeCheckingValidatorNe
+ "g = foreach f generate group, field1 + 1, field4 + 2.0 ;";
checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
- checkLastForeachCastLoadFunc(query, "PigStorage('b')", 2);
+ checkLastForeachCastLoadFunc(query, "org.apache.pig.test.PigStorageWithDifferentCaster('b')", 2);
}
@Test
public void testCogroupTopKLineageNoSchema() throws Throwable {
String query = "a = load 'a' using PigStorage('a');"
- + "b = load 'a' using PigStorage('b');"
+ + "b = load 'a' using org.apache.pig.test.PigStorageWithDifferentCaster('b');"
+ "c = cogroup a by $0, b by $0;"
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = order d by $2 desc;"
@@ -3496,8 +3496,13 @@ public class TestTypeCheckingValidatorNe
+ "d = foreach c generate group, flatten(a), flatten(b) ;"
+ "e = foreach d generate $0, $1 + 1, $2 + 2.0 ;";
- checkLastForeachCastLoadFunc(query, null, 1);
- checkLastForeachCastLoadFunc(query, null, 2);
+ // PigStorage and PigStreaming both returns Utf8StorageConverter so
+ // it may return either PigStorage or PigStreaming.
+ // Here, our code happens to return the first one thus
+ // checking with PigStorage. Depending on the implementation,
+ // this test may start failing.
+ checkLastForeachCastLoadFunc(query, "PigStorage('a')", 1);
+ checkLastForeachCastLoadFunc(query, "PigStorage('a')", 2);
}
@Test
Modified: pig/branches/tez/test/org/apache/pig/test/TestUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestUDF.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestUDF.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestUDF.java Thu Oct 31 19:21:28 2013
@@ -211,6 +211,21 @@ public class TestUDF {
pig.dumpSchema("c");
pig.dumpSchema("d");
}
+
+ @Test
+ public void testEvalFuncGetVarArgToFunc() throws Exception {
+ String input = "udf_test_jira_3444.txt";
+ Util.createLocalInputFile(input, new String[]{"dummy"});
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ pigServer.registerQuery("A = LOAD '"+input+"' USING PigStorage(',') AS (x:chararray);");
+ pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithVarArgs(3);");
+ pigServer.registerQuery("C = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithVarArgs(1,2,3,4);");
+
+ Iterator<Tuple> it = pigServer.openIterator("B");
+ assertEquals(Integer.valueOf(3),(Integer)it.next().get(0));
+ it = pigServer.openIterator("C");
+ assertEquals(Integer.valueOf(10), (Integer)it.next().get(0));
+ }
@Test(expected = FrontendException.class)
public void testEnsureProperSchema2() throws Exception {
@@ -272,4 +287,33 @@ public class TestUDF {
return l;
}
}
+
+ public static class UdfWithFuncSpecWithVarArgs extends EvalFunc<Integer> {
+ public UdfWithFuncSpecWithVarArgs() {}
+
+ @Override
+ public Integer exec(Tuple input) throws IOException {
+ int res = 0;
+ if (input == null || input.size() == 0) {
+ return res;
+ }
+ for (int i = 0; i < input.size(); i++) {
+ res += (Integer)input.get(i);
+ }
+ return res;
+ }
+
+ @Override
+ public SchemaType getSchemaType() {
+ return SchemaType.VARARG;
+ }
+
+ @Override
+ public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+ List<FuncSpec> l = new ArrayList<FuncSpec>();
+ Schema s1 = new Schema(new Schema.FieldSchema(null,DataType.INTEGER));
+ l.add(new FuncSpec(this.getClass().getName(), s1));
+ return l;
+ }
+ }
}
\ No newline at end of file
Modified: pig/branches/tez/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestUnion.java?rev=1537592&r1=1537591&r2=1537592&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestUnion.java Thu Oct 31 19:21:28 2013
@@ -278,4 +278,94 @@ public class TestUnion {
assertEquals(2, recordCount);
}
-}
\ No newline at end of file
+ @Test
+ public void testCastingAfterUnionWithMultipleLoadersDifferentCasters()
+ throws Exception {
+ // Note that different caster case only works when each field is still coming
+ // from the single Loader.
+ // In the case below, 'a' is coming from A(PigStorage)
+ // and 'b' is coming from B(TextLoader). No overlaps.
+ File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"1","2","3"});
+ File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"a","b","c"});
+
+ PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+ //PigStorage and TextLoader have different LoadCasters
+ ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);");
+ ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (b:bytearray);");
+ ps.registerQuery("C = union onschema A,B;");
+ ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
+
+ Schema dumpSchema = ps.dumpSchema("D");
+ Schema expected = new Schema ();
+ expected.add(new Schema.FieldSchema("a", DataType.INTEGER));
+ expected.add(new Schema.FieldSchema("b", DataType.CHARARRAY));
+ assertEquals(expected, dumpSchema);
+
+ Iterator<Tuple> itr = ps.openIterator("D");
+ int recordCount = 0;
+ while(itr.next() != null)
+ ++recordCount;
+ assertEquals(6, recordCount);
+
+ }
+
+ @Test
+ public void testCastingAfterUnionWithMultipleLoadersDifferentCasters2()
+ throws Exception {
+ // A bit more complicated pattern but still same requirement of each
+ // field coming from the same Loader.
+ // 'a' is coming from A(PigStorage)
+ // 'i' is coming from B and C but both from the TextLoader.
+ File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"b","c", "1", "3"});
+ File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"a","b","c"});
+ File f3 = Util.createInputFile("tmp", "i3.txt", new String[] {"1","2","3"});
+
+ PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+ ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);"); // Using PigStorage()
+ ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
+ ps.registerQuery("C = load '" + f3.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
+ ps.registerQuery("B2 = join B by i, A by a;"); //{A::a: bytearray,B::i: bytearray}
+ ps.registerQuery("B3 = foreach B2 generate a, B::i as i;"); //{A::a: bytearray,i: bytearray}
+ ps.registerQuery("C2 = join C by i, A by a;"); //{A::a: bytearray,C::i: bytearray}
+ ps.registerQuery("C3 = foreach C2 generate a, C::i as i;"); //{A::a: bytearray,i: bytearray}
+ ps.registerQuery("D = union onschema B3,C3;"); // {A::a: bytearray,i: bytearray}
+ ps.registerQuery("E = foreach D generate (chararray) a, (chararray) i;");//{A::a: chararray,i: chararray}
+ Iterator<Tuple> itr = ps.openIterator("E");
+ int recordCount = 0;
+ while(itr.next() != null)
+ ++recordCount;
+ assertEquals(4, recordCount);
+
+ }
+
+ @Test
+ public void testCastingAfterUnionWithMultipleLoadersSameCaster()
+ throws Exception {
+ // Fields coming from different loaders but
+ // having the same LoadCaster.
+ File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"1\ta","2\tb","3\tc"});
+ PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
+ // PigStorage and PigStorageWithStatistics have the same
+ // LoadCaster(== Utf8StorageConverter)
+ ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray, b:bytearray);");
+ ps.registerQuery("B = load '" + f1.getAbsolutePath() +
+ "' using org.apache.pig.test.PigStorageWithStatistics() as (a:bytearray, b:bytearray);");
+ ps.registerQuery("C = union onschema A,B;");
+ ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
+ // 'a' is coming from A and 'b' is coming from B; No overlaps.
+
+ Schema dumpSchema = ps.dumpSchema("D");
+ Schema expected = new Schema ();
+ expected.add(new Schema.FieldSchema("a", DataType.INTEGER));
+ expected.add(new Schema.FieldSchema("b", DataType.CHARARRAY));
+ assertEquals(expected, dumpSchema);
+
+ Iterator<Tuple> itr = ps.openIterator("D");
+ int recordCount = 0;
+ while(itr.next() != null)
+ ++recordCount;
+ assertEquals(6, recordCount);
+
+ }
+
+}
Propchange: pig/branches/tez/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2
------------------------------------------------------------------------------
Merged /pig/trunk/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2:r1532966-1537587