You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/03/18 03:19:25 UTC
svn commit: r1578718 [2/2] - in /pig/branches/tez: ./ conf/ src/
src/docs/src/documentation/content/xdocs/ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/datastorage/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/...
Modified: pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue Mar 18 02:19:24 2014
@@ -2168,9 +2168,11 @@ public class TestLogicalPlanBuilder {
LogicalPlanData lData = pigServer.getLogicalPlanData();
assertEquals("LoadFunc must be PigStorage", "org.apache.pig.builtin.PigStorage", lData.getLoadFuncs().get(0));
assertEquals("StoreFunc must be PigStorageWithSchema", "org.apache.pig.test.PigStorageWithSchema", lData.getStoreFuncs().get(0));
- assertEquals("Number of sources must be 2", lData.getSources().size(), 2);
+ assertEquals("Number of sources must be 2", lData.getNumSources(), 2);
+ assertEquals("Number of sinks must be 1", lData.getNumSinks(), 1);
assertTrue("Source must end with input.txt", lData.getSources().get(0).endsWith("input.txt"));
assertTrue("Sink must end with output", lData.getSinks().get(0).endsWith("output"));
+ assertEquals("Number of logical relational operators must be 4", lData.getNumLogicalRelationOperators(), 4);
}
/**
Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java Tue Mar 18 02:19:24 2014
@@ -1234,9 +1234,8 @@ public class TestMultiQueryCompiler {
+ "describe b;"
+ "store b into '/tmp/output1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -1257,9 +1256,8 @@ public class TestMultiQueryCompiler {
+ "illustrate b;"
+ "store b into '/tmp/output1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -1280,9 +1278,8 @@ public class TestMultiQueryCompiler {
+ "explain b;"
+ "store b into '/tmp/output1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -1303,9 +1300,8 @@ public class TestMultiQueryCompiler {
+ "dump b;"
+ "store b into '/tmp/output1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -1336,9 +1332,8 @@ public class TestMultiQueryCompiler {
public void testLoadStoreLoop() {
try {
String script = "a = load 'dummy'; b = filter a by $0 == 1; store b into 'dummy';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
myPig.getPigContext().inExplain = true;
parser.parseStopOnError();
} catch (Exception e) {
Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java Tue Mar 18 02:19:24 2014
@@ -443,9 +443,8 @@ public class TestMultiQueryLocal {
+ "explain b;"
+ "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -468,9 +467,8 @@ public class TestMultiQueryLocal {
+ "dump b;"
+ "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -493,9 +491,8 @@ public class TestMultiQueryLocal {
+ "describe b;"
+ "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
@@ -518,10 +515,9 @@ public class TestMultiQueryLocal {
+ "illustrate b;"
+ "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), myPig);
parser.setInteractive(false);
myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "true");
- parser.setParams(myPig);
parser.parseStopOnError();
} catch (Exception e) {
Modified: pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Tue Mar 18 02:19:24 2014
@@ -1111,6 +1111,25 @@ public class TestNewPlanPushDownForeachF
((LOLoad)load).getSchema().getField("a1") != null );
}
+ // See PIG-3782
+ @Test
+ public void testForeachJoinWithUserDefinedSchemaAndPruning() throws Exception {
+ String query =
+ "a = load '1.txt' as (a0:int, a1, a2:bag{});" +
+ "b = load '2.txt' as (b0:int, b1);" +
+ "c = foreach a generate a0, flatten(a2) as (q1, q2);" +
+ "d = join c by a0, b by b0;" +
+ "e = foreach d generate a0, q1, q2;" +
+ "f = foreach e generate a0, (int)q1, (int)q2;" +
+ "store f into 'output';" ;
+
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlanWithPruning( query );
+ //In the original issue, Exception is thrown during the Pruning due to
+ //incorrect UID assignment by the PushDownForEachFlatten failing this
+ //test
+
+ }
+
public class MyPlanOptimizerWithPruning extends LogicalPlanOptimizer {
protected MyPlanOptimizerWithPruning (OperatorPlan p, int iterations) {
super(p, iterations, new HashSet<String>());
Modified: pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java Tue Mar 18 02:19:24 2014
@@ -52,7 +52,7 @@ import org.apache.pig.tools.pigstats.Out
import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
import org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil;
import org.junit.AfterClass;
@@ -193,7 +193,7 @@ public class TestPigRunner {
String[] args = { "-Dstop.on.failure=true", "-Dopt.multiquery=false", "-Daggregate.warning=false", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
- assertTrue(stats instanceof SimpleFetchPigStats);
+ assertTrue(stats instanceof EmptyPigStats);
assertTrue(stats.isSuccessful());
assertEquals(0, stats.getNumberJobs());
assertEquals(stats.getJobGraph().size(), 0);
Modified: pig/branches/tez/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPigServer.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPigServer.java Tue Mar 18 02:19:24 2014
@@ -958,9 +958,8 @@ public class TestPigServer {
PigServer pigServer = new PigServer(pigContext);
data = resetData(pigServer);
data.set("foo", tuple("a", 1, "b"), tuple("b", 2, "c"), tuple("c", 3, "d"));
- GruntParser grunt = new GruntParser(in);
+ GruntParser grunt = new GruntParser(in, pigServer);
grunt.setInteractive(false);
- grunt.setParams(pigServer);
grunt.parseStopOnError(true); //not batch
}
Modified: pig/branches/tez/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/Util.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/Util.java Tue Mar 18 02:19:24 2014
@@ -613,10 +613,8 @@ public class Util {
}
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
String script = getMkDirCommandForHadoop2_0(fileNameOnCluster) + "fs -put " + localFileName + " " + fileNameOnCluster;
-
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), ps);
parser.setInteractive(false);
- parser.setParams(ps);
try {
parser.parseStopOnError();
} catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
@@ -635,9 +633,8 @@ public class Util {
new File(toLocalFileName).deleteOnExit();
- GruntParser parser = new GruntParser(new StringReader(script));
+ GruntParser parser = new GruntParser(new StringReader(script), ps);
parser.setInteractive(false);
- parser.setParams(ps);
try {
parser.parseStopOnError();
} catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
@@ -1289,4 +1286,19 @@ public class Util {
}
};
}
+
+ /**
+ *
+ * @param expected
+ * Exception class that is expected to be thrown
+ * @param found
+ * Exception that occurred in the test
+ * @param message
+ * expected String to verify against
+ */
+ public static void assertExceptionAndMessage(Class<?> expected,
+ Exception found, String message) {
+ assertEquals(expected, found.getClass());
+ assertEquals(found.getMessage(), message);
+ }
}
Propchange: pig/branches/tez/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2
------------------------------------------------------------------------------
Merged /pig/trunk/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2:r1571422-1578679
Modified: pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java Tue Mar 18 02:19:24 2014
@@ -19,7 +19,10 @@ import java.util.Map;
import junit.framework.Assert;
+import org.apache.pig.ExecType;
+import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.pigunit.pig.GruntParser;
+import org.apache.pig.pigunit.pig.PigServer;
import org.junit.Before;
import org.junit.Test;
@@ -30,12 +33,13 @@ public class TestGruntParser {
@SuppressWarnings("serial")
@Before
- public void setUp() {
+ public void setUp() throws ExecException {
override = new HashMap<String, String>() {{
put("STORE", "");
put("DUMP", "");
}};
- parser = new GruntParser(new StringReader(""), override);
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ parser = new GruntParser(new StringReader(""), pigServer, override);
}
@Test
Modified: pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java Tue Mar 18 02:19:24 2014
@@ -39,18 +39,17 @@ public class TestGruntParser {
public void setup() throws IOException {
pig = new PigServer(ExecType.LOCAL);
}
-
+
@Test
public void testProcessRemove() throws IOException {
File tmpFile = File.createTempFile("TestGruntParser", "testProcessRemove");
String dummyScript = "";
- parser = new GruntParser(new ByteArrayInputStream(dummyScript.getBytes()));
- parser.setParams(pig);
-
+ parser = new GruntParser(new ByteArrayInputStream(dummyScript.getBytes()), pig);
+
//Delete existing file and check that it doesn't exist
parser.processRemove(tmpFile.getAbsolutePath(), "");
assertFalse(tmpFile.exists());
-
+
//Delete non-existing file and check for exception
try {
parser.processRemove(tmpFile.getAbsolutePath(), "");
Modified: pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl (original)
+++ pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl Tue Mar 18 02:19:24 2014
@@ -80,7 +80,10 @@ for(my $i = 1; $i <= 17; $i++) {
$mr_times = int($mr_times + 0.5);
$total_mr_times = $total_mr_times + $mr_times;
- my $multiplier = $pig_times/$mr_times;
+
+ my $multiplier=0;
+ if ($mr_times!=0)
+ $multiplier = $pig_times/$mr_times;
print "PigMix_$i pig run time: $pig_times, java run time: $mr_times, multiplier: $multiplier\n";
}
}