You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/30 21:07:29 UTC
svn commit: r1598702 [18/23] - in /pig/trunk: ./ ivy/
shims/src/hadoop23/org/apache/pig/backend/hadoop23/
shims/test/hadoop20/org/apache/pig/test/
shims/test/hadoop23/org/apache/pig/test/ src/META-INF/services/
src/org/apache/pig/ src/org/apache/pig/ba...
Modified: pig/trunk/test/org/apache/pig/test/TestForEachNestedPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestForEachNestedPlan.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestForEachNestedPlan.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestForEachNestedPlan.java Fri May 30 19:07:23 2014
@@ -18,7 +18,7 @@
package org.apache.pig.test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.FileWriter;
@@ -29,9 +29,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
-import junit.framework.Assert;
-
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
@@ -39,32 +36,33 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.parser.ParserException;
import org.apache.pig.test.utils.TestHelper;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Test;
public class TestForEachNestedPlan {
- static MiniCluster cluster = MiniCluster.buildCluster();
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pig ;
public TestForEachNestedPlan() throws Throwable {
- pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()) ;
+ pig = new PigServer(cluster.getExecType(), cluster.getProperties()) ;
}
Boolean[] nullFlags = new Boolean[]{ false, true };
-
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
@Test
public void testInnerOrderBy() throws Exception {
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Running testInnerOrderBy with nullFlags set to :"
+ nullFlags[i]);
File tmpFile = genDataSetFile1(nullFlags[i]);
- pig.registerQuery("a = load '"
+ pig.registerQuery("a = load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "'; ");
pig.registerQuery("b = group a by $0; ");
pig.registerQuery("c = foreach b { " + " c1 = order $1 by *; "
@@ -82,9 +80,9 @@ public class TestForEachNestedPlan {
}
@Test
- public void testInnerOrderByStarWithSchema() throws Exception {
+ public void testInnerOrderByStarWithSchema() throws Exception {
File tmpFile = genDataSetFile1(false);
- pig.registerQuery("a = load '" + Util.generateURI(tmpFile.toString(),
+ pig.registerQuery("a = load '" + Util.generateURI(tmpFile.toString(),
pig.getPigContext()) + "' as (a0, a1);");
pig.registerQuery("b = group a by a0; ");
pig.registerQuery("c = foreach b { d = order a by *; "
@@ -99,23 +97,23 @@ public class TestForEachNestedPlan {
}
Assert.assertEquals(count, 10);
}
-
+
@Test
- public void testMultiColInAlias() throws Exception {
+ public void testMultiColInAlias() throws Exception {
pig.getPigContext().getProperties().setProperty("pig.exec.nosecondarykey", "true");
String INPUT_FILE = "test-multi-alias.txt";
PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
w.println("10\tnrai01\t01");
w.println("20\tnrai02\t02");
w.close();
-
+
try {
-
+
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
pig.registerQuery("A = load '" + INPUT_FILE + "' "
+ "as (a:int, b:chararray, c:int);");
pig.registerQuery("B = GROUP A BY (a, b);") ;
-
+
DataBag dbfrj = BagFactory.getInstance().newDefaultBag(), dbshj = BagFactory.getInstance().newDefaultBag();
{
pig.registerQuery("C = FOREACH B { bg = A.($1,$2); GENERATE group, bg; } ;") ;
@@ -132,7 +130,7 @@ public class TestForEachNestedPlan {
}
}
Assert.assertEquals(dbfrj.size(), dbshj.size());
- Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
+ Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbshj));
} finally{
new File(INPUT_FILE).delete();
@@ -144,9 +142,9 @@ public class TestForEachNestedPlan {
}
}
}
-
+
@Test
- public void testAlgebricFuncWithoutGroupBy()
+ public void testAlgebricFuncWithoutGroupBy()
throws IOException, ParserException {
String INPUT_FILE = "test-sum.txt";
@@ -194,7 +192,7 @@ public class TestForEachNestedPlan {
}
@Test
- public void testInnerDistinct()
+ public void testInnerDistinct()
throws IOException, ParserException {
String INPUT_FILE = "test-distinct.txt";
@@ -208,13 +206,13 @@ public class TestForEachNestedPlan {
try {
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
-
+
pig.registerQuery("A = load '" + INPUT_FILE
+ "' as (age:int, gpa:int);");
pig.registerQuery("B = group A by age;");
pig.registerQuery("C = foreach B { D = A.gpa; E = distinct D; " +
"generate group, MIN(E); };");
-
+
Iterator<Tuple> iter = pig.openIterator("C");
List<Tuple> expectedResults =
@@ -224,9 +222,9 @@ public class TestForEachNestedPlan {
int counter = 0;
while (iter.hasNext()) {
assertEquals(expectedResults.get(counter++).toString(),
- iter.next().toString());
+ iter.next().toString());
}
-
+
assertEquals(expectedResults.size(), counter);
} finally{
new File(INPUT_FILE).delete();
@@ -240,7 +238,7 @@ public class TestForEachNestedPlan {
}
@Test
- public void testInnerOrderByAliasReuse()
+ public void testInnerOrderByAliasReuse()
throws IOException, ParserException {
String INPUT_FILE = "test-innerorderbyaliasreuse.txt";
@@ -253,13 +251,13 @@ public class TestForEachNestedPlan {
try {
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
-
+
pig.registerQuery("A = load '" + INPUT_FILE
+ "' as (v1:int, v2:int);");
pig.registerQuery("B = group A by v1;");
pig.registerQuery("C = foreach B { X = A; X = order X by v2 asc; " +
"generate flatten(X); };");
-
+
Iterator<Tuple> iter = pig.openIterator("C");
List<Tuple> expectedResults =
@@ -269,9 +267,9 @@ public class TestForEachNestedPlan {
int counter = 0;
while (iter.hasNext()) {
assertEquals(expectedResults.get(counter++).toString(),
- iter.next().toString());
+ iter.next().toString());
}
-
+
assertEquals(expectedResults.size(), counter);
} finally{
new File(INPUT_FILE).delete();
@@ -283,8 +281,8 @@ public class TestForEachNestedPlan {
}
}
}
-
-
+
+
/***
* For generating a sample dataset
*/
@@ -296,7 +294,7 @@ public class TestForEachNestedPlan {
DecimalFormat formatter = new DecimalFormat("0000000");
Random r = new Random();
-
+
for (int i = 0; i < dataLength; i++) {
data[i] = new String[2] ;
// inject nulls randomly
Modified: pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGrunt.java Fri May 30 19:07:23 2014
@@ -41,6 +41,7 @@ import org.apache.log4j.FileAppender;
import org.apache.log4j.Level;
import org.apache.log4j.PatternLayout;
import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
@@ -53,17 +54,18 @@ import org.apache.pig.test.Util.ProcessR
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestGrunt {
- static MiniCluster cluster = MiniCluster.buildCluster();
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private String basedir = "test/org/apache/pig/test/data";
@BeforeClass
public static void oneTimeSetup() throws Exception {
- cluster.setProperty("opt.multiquery","true");
+ cluster.setProperty(PigConfiguration.OPT_MULTIQUERY,"true");
}
@AfterClass
@@ -71,10 +73,15 @@ public class TestGrunt {
cluster.shutDown();
}
+ @Before
+ public void setup() {
+ Util.resetStateForExecModeSwitch();
+ }
+
@Test
public void testCopyFromLocal() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "copyFromLocal README.txt sh_copy ;";
@@ -90,7 +97,7 @@ public class TestGrunt {
@Test
public void testDefine() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "define myudf org.apache.pig.builtin.AVG();\n";
@@ -110,7 +117,7 @@ public class TestGrunt {
@Test
public void testBagSchema() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1' as (b: bag{t:(i: int, c:chararray, f: float)});\n";
@@ -125,7 +132,7 @@ public class TestGrunt {
@Test
public void testBagSchemaFail() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1'as (b: bag{t:(i: int, c:chararray, f: float)});\n";
@@ -146,7 +153,7 @@ public class TestGrunt {
@Test
public void testBagConstant() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1', 0.4f),(2, '2', 0.45)};\n";
@@ -161,7 +168,7 @@ public class TestGrunt {
@Test
public void testBagConstantWithSchema() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1'; b = foreach a generate "
@@ -178,7 +185,7 @@ public class TestGrunt {
@Test
public void testBagConstantInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1'; "
@@ -194,7 +201,7 @@ public class TestGrunt {
@Test
public void testBagConstantWithSchemaInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'input1'; "
@@ -211,7 +218,7 @@ public class TestGrunt {
@Test
public void testParsingAsInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast); "
@@ -228,7 +235,7 @@ public class TestGrunt {
@Test
public void testParsingAsInForeachWithOutBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast); "
@@ -244,7 +251,7 @@ public class TestGrunt {
@Test
public void testParsingWordWithAsInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast); "
@@ -260,7 +267,7 @@ public class TestGrunt {
@Test
public void testParsingWordWithAsInForeachWithOutBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast); "
@@ -276,7 +283,7 @@ public class TestGrunt {
@Test
public void testParsingWordWithAsInForeachWithOutBlock2() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "cash = load 'foo' as (foo, fast); "
@@ -293,7 +300,7 @@ public class TestGrunt {
@Test
public void testParsingGenerateInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
@@ -309,7 +316,7 @@ public class TestGrunt {
@Test
public void testParsingGenerateInForeachWithOutBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
@@ -325,7 +332,7 @@ public class TestGrunt {
@Test
public void testParsingAsGenerateInForeachBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
@@ -344,7 +351,7 @@ public class TestGrunt {
@Test
public void testParsingAsGenerateInForeachWithOutBlock() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
@@ -363,7 +370,7 @@ public class TestGrunt {
@Test
public void testRunStatment() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate);"
@@ -380,7 +387,7 @@ public class TestGrunt {
@Test
public void testExecStatment() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
boolean caught = false;
@@ -404,7 +411,7 @@ public class TestGrunt {
@Test
public void testRunStatmentNested() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); run "
@@ -420,7 +427,7 @@ public class TestGrunt {
@Test
public void testExecStatmentNested() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
boolean caught = false;
@@ -443,7 +450,7 @@ public class TestGrunt {
@Test
public void testErrorLineNumber() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "A = load 'x' as ( u:int, v:chararray );\n" +
@@ -468,7 +475,7 @@ public class TestGrunt {
@Test
public void testExplainEmpty() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); run "
@@ -484,7 +491,7 @@ public class TestGrunt {
@Test
public void testExplainScript() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -script "
@@ -509,7 +516,7 @@ public class TestGrunt {
*/
@Test
public void testExplainScriptIsEachStatementValidated() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate);" +
@@ -647,7 +654,7 @@ public class TestGrunt {
@Test
public void testExplainScript2() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "explain -script "
@@ -692,7 +699,7 @@ public class TestGrunt {
@Test
public void testExplainBrief() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -brief -script "
@@ -708,7 +715,7 @@ public class TestGrunt {
@Test
public void testExplainDot() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -dot -script "
@@ -724,7 +731,7 @@ public class TestGrunt {
@Test
public void testExplainOut() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -out /tmp -script "
@@ -740,7 +747,7 @@ public class TestGrunt {
@Test
public void testPartialExecution() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
FileLocalizer.setInitialized(false);
@@ -761,7 +768,7 @@ public class TestGrunt {
@Test
public void testFileCmds() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd =
@@ -792,7 +799,7 @@ public class TestGrunt {
@Test
public void testCD() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd =
@@ -817,7 +824,7 @@ public class TestGrunt {
@Test
public void testDump() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd =
@@ -841,7 +848,8 @@ public class TestGrunt {
@Test
public void testIllustrate() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ if (!Util.isMapredExecType(cluster.getExecType())) return;
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd =
@@ -865,7 +873,8 @@ public class TestGrunt {
@Test
public void testKeepGoing() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ if (!Util.isMapredExecType(cluster.getExecType())) return;
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
@@ -895,7 +904,7 @@ public class TestGrunt {
@Test
public void testKeepGoigFailed() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "passwd");
String strCmd =
@@ -952,7 +961,8 @@ public class TestGrunt {
@Test
public void testStopOnFailure() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ if (!Util.isMapredExecType(cluster.getExecType())) return;
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
context.getProperties().setProperty("stop.on.failure", ""+true);
@@ -989,7 +999,7 @@ public class TestGrunt {
@Test
public void testFsCommand() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE,cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(),cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd =
@@ -1017,7 +1027,7 @@ public class TestGrunt {
public void testShellCommand(){
try {
- PigServer server = new PigServer(ExecType.MAPREDUCE,cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(),cluster.getProperties());
PigContext context = server.getPigContext();
String strRemoveFile = "rm";
@@ -1142,7 +1152,7 @@ public class TestGrunt {
@Test
public void testSetPriority() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "set job.priority high\n";
@@ -1158,7 +1168,7 @@ public class TestGrunt {
@Test
public void testSetWithQuotes() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "set job.priority 'high'\n";
@@ -1174,7 +1184,7 @@ public class TestGrunt {
@Test
public void testRegisterWithQuotes() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "register 'pig-withouthadoop.jar'\n";
@@ -1191,7 +1201,7 @@ public class TestGrunt {
@Test
public void testRegisterWithoutQuotes() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "register pig-withouthadoop.jar\n";
@@ -1217,7 +1227,7 @@ public class TestGrunt {
Util.createLocalInputFile( "testRegisterScripts.py", script);
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "register testRegisterScripts.py using jython as pig\n";
@@ -1255,7 +1265,7 @@ public class TestGrunt {
// than an unrelated EOF error message
@Test
public void testBlockErrMessage() throws Throwable {
- PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
String script = "A = load 'inputdata' using PigStorage() as ( curr_searchQuery );\n" +
Modified: pig/trunk/test/org/apache/pig/test/TestImplicitSplit.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestImplicitSplit.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestImplicitSplit.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestImplicitSplit.java Fri May 30 19:07:23 2014
@@ -19,13 +19,13 @@ package org.apache.pig.test;
import static org.apache.pig.test.utils.TypeCheckingTestUtil.printTypeGraph;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.util.HashMap;
import java.util.Iterator;
-
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
@@ -37,22 +37,22 @@ import org.junit.Test;
public class TestImplicitSplit {
private PigServer pigServer;
- static MiniCluster cluster = MiniCluster.buildCluster();
-
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+
@Before
public void setUp() throws Exception {
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
}
@After
public void tearDown() throws Exception {
}
-
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
@Test
public void testImplicitSplit() throws Exception{
int LOOP_SIZE = 20;
@@ -76,20 +76,20 @@ public class TestImplicitSplit {
assertEquals(20, cnt);
Util.deleteFile(cluster, inputFileName);
}
-
+
@Test
public void testImplicitSplitInCoGroup() throws Exception {
// this query is similar to the one reported in JIRA - PIG-537
// Create input file
String input1 = "testImplicitSplitInCoGroup-input1.txt";
String input2 = "testImplicitSplitInCoGroup-input2.txt";
- Util.createInputFile(cluster, input1,
+ Util.createInputFile(cluster, input1,
new String[] {"a:1", "b:2", "b:20", "c:3", "c:30"});
- Util.createInputFile(cluster, input2,
+ Util.createInputFile(cluster, input2,
new String[] {"a:first", "b:second", "c:third"});
- pigServer.registerQuery("a = load '" + input1 +
+ pigServer.registerQuery("a = load '" + input1 +
"' using PigStorage(':') as (name:chararray, marks:int);");
- pigServer.registerQuery("b = load '" + input2 +
+ pigServer.registerQuery("b = load '" + input2 +
"' using PigStorage(':') as (name:chararray, rank:chararray);");
pigServer.registerQuery("c = cogroup a by name, b by name;");
pigServer.registerQuery("d = foreach c generate group, FLATTEN(a.marks) as newmarks;");
@@ -101,7 +101,7 @@ public class TestImplicitSplit {
results.put(3, new Object[] { "c", 3, "c", 3 });
results.put(20, new Object[] { "b", 20, "b", 20 });
results.put(30, new Object[] { "c", 30, "c", 30 });
-
+
Iterator<Tuple> it = pigServer.openIterator("f");
while(it.hasNext()) {
Tuple t = it.next();
@@ -109,13 +109,13 @@ public class TestImplicitSplit {
Integer group = (Integer)t.get(0);
Object[] groupValues = results.get(group);
for(int i = 0; i < 4; i++) {
- assertEquals(groupValues[i], t.get(i+1));
+ assertEquals(groupValues[i], t.get(i+1));
}
}
Util.deleteFile(cluster, input1);
Util.deleteFile(cluster, input2);
}
-
+
@Test
public void testImplicitSplitInCoGroup2() throws Exception {
// this query is similar to the one reported in JIRA - PIG-537
@@ -127,12 +127,12 @@ public class TestImplicitSplit {
"f = foreach e generate group, flatten(a), flatten(d);" +
"store f into 'output';";
org.apache.pig.newplan.logical.relational.LogicalPlan plan = Util.buildLp(pigServer, query);
-
+
printTypeGraph(plan) ;
-
+
// this will run ImplicitSplitInserter
Util.optimizeNewLP(plan);
-
+
// get Schema of leaf and compare:
String schema = "group:int,a::name:chararray,a::marks:int,d::group:chararray,d::newmarks:int";
LogicalSchema sch = ((LogicalRelationalOperator)plan.getSinks().get(0)).getSchema();
Modified: pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java Fri May 30 19:07:23 2014
@@ -30,7 +30,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
@@ -55,15 +54,15 @@ import org.junit.Before;
import org.junit.Test;
public class TestInputOutputMiniClusterFileValidator {
- private static MiniCluster cluster = MiniCluster.buildCluster();
+ private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pig;
private PigContext ctx;
@Before
public void setUp() throws Exception {
- ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
+ ctx = new PigContext(cluster.getExecType(), cluster.getProperties());
ctx.connect() ;
- pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pig = new PigServer(ctx);
}
@AfterClass
@@ -165,7 +164,6 @@ public class TestInputOutputMiniClusterF
@Test
public void testValidationNeg() throws Throwable{
- PigServer pig = new PigServer(ExecType.MAPREDUCE,cluster.getProperties());
try{
pig.setBatchOn();
pig.registerQuery("A = load 'inputfile' using PigStorage () as (a:int);");
Modified: pig/trunk/test/org/apache/pig/test/TestJoin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJoin.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJoin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJoin.java Fri May 30 19:07:23 2014
@@ -40,7 +40,6 @@ import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.LogUtils;
import org.apache.pig.impl.util.Utils;
@@ -60,12 +59,12 @@ import com.google.common.collect.Sets;
public class TestJoin {
- static MiniCluster cluster;
+ private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pigServer;
TupleFactory mTf = TupleFactory.getInstance();
BagFactory mBf = BagFactory.getInstance();
- ExecType[] execTypes = new ExecType[] {ExecType.LOCAL, ExecType.MAPREDUCE};
+ private static ExecType[] execTypes = new ExecType[] {ExecType.LOCAL, cluster.getExecType()};
@AfterClass
public static void oneTimeTearDown() throws Exception {
@@ -73,12 +72,9 @@ public class TestJoin {
}
private void setUp(ExecType execType) throws ExecException {
- // cause a reinitialization of FileLocalizer's
- // internal state
- FileLocalizer.setInitialized(false);
- if(execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ Util.resetStateForExecModeSwitch();
+ if(execType == cluster.getExecType()) {
+ pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
} else if(execType == ExecType.LOCAL) {
pigServer = new PigServer(ExecType.LOCAL);
}
@@ -86,7 +82,7 @@ public class TestJoin {
private String createInputFile(ExecType execType, String fileNameHint, String[] data) throws IOException {
String fileName = "";
- if(execType == ExecType.MAPREDUCE) {
+ if(execType == cluster.getExecType()) {
Util.createInputFile(cluster, fileNameHint, data);
fileName = fileNameHint;
} else if (execType == ExecType.LOCAL) {
@@ -97,7 +93,7 @@ public class TestJoin {
}
private void deleteInputFile(ExecType execType, String fileName) throws IOException {
- if(execType == ExecType.MAPREDUCE) {
+ if(execType == cluster.getExecType()) {
Util.deleteFile(cluster, fileName);
} else if(execType == ExecType.LOCAL){
fileName = fileName.replace("file://", "");
@@ -107,7 +103,7 @@ public class TestJoin {
@Test
public void testJoinWithMissingFieldsInTuples() throws IOException{
- setUp(ExecType.MAPREDUCE);
+ setUp(cluster.getExecType());
String[] input1 = {
"ff ff ff",
"",
@@ -126,16 +122,16 @@ public class TestJoin {
""
};
- String firstInput = createInputFile(ExecType.MAPREDUCE, "a.txt", input1);
- String secondInput = createInputFile(ExecType.MAPREDUCE, "b.txt", input2);
+ String firstInput = createInputFile(cluster.getExecType(), "a.txt", input1);
+ String secondInput = createInputFile(cluster.getExecType(), "b.txt", input2);
String script = "a = load 'a.txt' using PigStorage(' ');" +
"b = load 'b.txt' using PigStorage('\u0001');" +
"c = join a by $0, b by $0;";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
assertFalse(it.hasNext());
- deleteInputFile(ExecType.MAPREDUCE, firstInput);
- deleteInputFile(ExecType.MAPREDUCE, secondInput);
+ deleteInputFile(cluster.getExecType(), firstInput);
+ deleteInputFile(cluster.getExecType(), secondInput);
}
@Test
@@ -582,13 +578,12 @@ public class TestJoin {
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
- assertTrue(it.hasNext());
- Tuple t = it.next();
- assertEquals("((1,a),(1,b))", t.toString());
-
- assertTrue(it.hasNext());
- t = it.next();
- assertEquals("((2,aa),(2,bb))", t.toString());
+ List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
+ new String[] {
+ "((1,'a'),(1,'b'))",
+ "((2,'aa'),(2,'bb'))"
+ });
+ Util.checkQueryOutputsAfterSort(it, expectedResults);
deleteInputFile(execType, firstInput);
deleteInputFile(execType, secondInput);
@@ -617,11 +612,9 @@ public class TestJoin {
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
- assertTrue(it.hasNext());
- Tuple t = it.next();
- assertEquals("(2,aa,2,aa)", t.toString());
-
- assertFalse(it.hasNext());
+ List<Tuple> expectedResults = Util
+ .getTuplesFromConstantTupleStrings(new String[] { "(2,'aa',2,'aa')" });
+ Util.checkQueryOutputs(it, expectedResults);
deleteInputFile(execType, firstInput);
deleteInputFile(execType, secondInput);
Modified: pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJoinSmoke.java Fri May 30 19:07:23 2014
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.Iterator;
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.BagFactory;
@@ -43,11 +42,10 @@ public class TestJoinSmoke {
private static final String SKEW_INPUT_FILE5 = "SkewedJoinInput5.txt";
private PigServer pigServer;
- private static MiniCluster cluster = MiniCluster.buildCluster();
- private File tmpFile;
+ private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
public TestJoinSmoke() throws ExecException, IOException {
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
}
@Before
Modified: pig/trunk/test/org/apache/pig/test/TestKeyTypeDiscoveryVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestKeyTypeDiscoveryVisitor.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestKeyTypeDiscoveryVisitor.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestKeyTypeDiscoveryVisitor.java Fri May 30 19:07:23 2014
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertFal
import java.util.Iterator;
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.Tuple;
@@ -38,7 +37,7 @@ import org.junit.Test;
*/
public class TestKeyTypeDiscoveryVisitor {
- static MiniCluster cluster = MiniCluster.buildCluster();
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pigServer;
TupleFactory mTf = TupleFactory.getInstance();
@@ -46,7 +45,7 @@ public class TestKeyTypeDiscoveryVisitor
@Before
public void setUp() throws Exception{
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); //TODO this doesn't need to be M/R mode
+ pigServer = new PigServer(cluster.getExecType(), cluster.getProperties()); //TODO this doesn't need to be M/R mode
}
@AfterClass
Modified: pig/trunk/test/org/apache/pig/test/TestLoad.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLoad.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLoad.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLoad.java Fri May 30 19:07:23 2014
@@ -35,6 +35,7 @@ import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.datastorage.DataStorage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -64,18 +65,18 @@ public class TestLoad {
PigContext pc;
PigServer[] servers;
-
- static MiniCluster cluster = MiniCluster.buildCluster();
-
+
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+
@Before
public void setUp() throws Exception {
FileLocalizer.deleteTempFiles();
- servers = new PigServer[] {
- new PigServer(ExecType.MAPREDUCE, cluster.getProperties()),
+ servers = new PigServer[] {
+ new PigServer(cluster.getExecType(), cluster.getProperties()),
new PigServer(ExecType.LOCAL, new Properties())
- };
+ };
}
-
+
@Test
public void testGetNextTuple() throws IOException {
pc = servers[0].getPigContext();
@@ -88,10 +89,10 @@ public class TestLoad {
POLoad ld = GenPhyOp.topLoadOp();
ld.setLFile(inpFSpec);
ld.setPc(pc);
-
+
DataBag inpDB = DefaultBagFactory.getInstance().newDefaultBag();
BufferedReader br = new BufferedReader(new FileReader("test/org/apache/pig/test/data/InputFiles/passwd"));
-
+
for(String line = br.readLine();line!=null;line=br.readLine()){
String[] flds = line.split(":",-1);
Tuple t = new DefaultTuple();
@@ -113,7 +114,7 @@ public class TestLoad {
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
@Test
public void testLoadRemoteRel() throws Exception {
for (PigServer pig : servers) {
@@ -144,7 +145,7 @@ public class TestLoad {
pc = servers[0].getPigContext();
boolean noConversionExpected = true;
checkLoadPath("hdfs:/tmp/test","hdfs:/tmp/test", noConversionExpected);
-
+
// check if a location 'hdfs:<abs path>' can actually be read using PigStorage
String[] inputFileNames = new String[] {
"/tmp/TestLoad-testLoadRemoteAbsSchema-input.txt"};
@@ -198,18 +199,18 @@ public class TestLoad {
boolean noConversionExpected = true;
checkLoadPath("hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3",
"hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3", noConversionExpected );
-
- // check if a location 'hdfs:<abs path>,hdfs:<abs path>' can actually be
+
+ // check if a location 'hdfs:<abs path>,hdfs:<abs path>' can actually be
// read using PigStorage
String[] inputFileNames = new String[] {
"/tmp/TestLoad-testCommaSeparatedString3-input1.txt",
"/tmp/TestLoad-testCommaSeparatedString3-input2.txt"};
- String inputString = "hdfs:" + inputFileNames[0] + ",hdfs:" +
+ String inputString = "hdfs:" + inputFileNames[0] + ",hdfs:" +
inputFileNames[1];
testLoadingMultipleFiles(inputFileNames, inputString);
-
+
}
-
+
@Test
public void testCommaSeparatedString4() throws Exception {
for (PigServer pig : servers) {
@@ -224,12 +225,12 @@ public class TestLoad {
pc = pig.getPigContext();
checkLoadPath("/usr/pig/{a,c},usr/pig/b","/usr/pig/{a,c},/tmp/usr/pig/b");
}
-
- // check if a location '<abs path>,<relative path>' can actually be
+
+ // check if a location '<abs path>,<relative path>' can actually be
// read using PigStorage
String loadLocationString = "/tmp/TestLoad-testCommaSeparatedStringMixed-input{1,2}.txt," +
"TestLoad-testCommaSeparatedStringMixed-input3.txt"; // current working dir is set to /tmp in checkLoadPath()
-
+
String[] inputFileNames = new String[] {
"/tmp/TestLoad-testCommaSeparatedStringMixed-input1.txt",
"/tmp/TestLoad-testCommaSeparatedStringMixed-input2.txt",
@@ -237,7 +238,7 @@ public class TestLoad {
pc = servers[0].getPigContext(); // test in map reduce mode
testLoadingMultipleFiles(inputFileNames, loadLocationString);
}
-
+
@Test
public void testCommaSeparatedString6() throws Exception {
for (PigServer pig : servers) {
@@ -245,7 +246,7 @@ public class TestLoad {
checkLoadPath("usr/pig/{a,c},/usr/pig/b","/tmp/usr/pig/{a,c},/usr/pig/b");
}
}
-
+
@Test
public void testNonDfsLocation() throws Exception {
String nonDfsUrl = "har:///user/foo/f.har";
@@ -256,11 +257,11 @@ public class TestLoad {
nonDfsUrl = nonDfsUrl.replaceFirst("/$", "");
assertEquals(nonDfsUrl, load.getFileSpec().getFileName());
}
-
+
@SuppressWarnings("unchecked")
- private void testLoadingMultipleFiles(String[] inputFileNames,
+ private void testLoadingMultipleFiles(String[] inputFileNames,
String loadLocationString) throws IOException, ParserException {
-
+
String[][] inputStrings = new String[][] {
new String[] { "hello\tworld"},
new String[] { "bye\tnow"},
@@ -270,7 +271,7 @@ public class TestLoad {
(Tuple) Util.getPigConstant("('hello', 'world')"),
(Tuple) Util.getPigConstant("('bye', 'now')"),
(Tuple) Util.getPigConstant("('all', 'good')")});
-
+
List<Tuple> expectedBasedOnNumberOfInputs = new ArrayList<Tuple>();
for(int i = 0; i < inputFileNames.length; i++) {
Util.createInputFile(pc, inputFileNames[i], inputStrings[i]);
@@ -280,7 +281,7 @@ public class TestLoad {
servers[0].registerQuery(" a = load '" + loadLocationString + "' as " +
"(s1:chararray, s2:chararray);");
Iterator<Tuple> it = servers[0].openIterator("a");
-
+
List<Tuple> actual = new ArrayList<Tuple>();
while(it.hasNext()) {
actual.add(it.next());
@@ -294,40 +295,40 @@ public class TestLoad {
}
}
}
-
+
private void checkLoadPath(String orig, String expected) throws Exception {
checkLoadPath(orig, expected, false);
}
- private void checkLoadPath(String orig, String expected,
+ private void checkLoadPath(String orig, String expected,
boolean noConversionExpected) throws Exception {
-
+
boolean[] multiquery = {true, false};
-
+
for (boolean b : multiquery) {
- pc.getProperties().setProperty("opt.multiquery", "" + b);
-
+ pc.getProperties().setProperty(PigConfiguration.OPT_MULTIQUERY, "" + b);
+
DataStorage dfs = pc.getDfs();
dfs.setActiveContainer(dfs.asContainer("/tmp"));
Map<String, String> fileNameMap = new HashMap<String, String>();
-
+
QueryParserDriver builder = new QueryParserDriver(pc, "Test-Load", fileNameMap);
-
+
String query = "a = load '"+orig+"';";
LogicalPlan lp = builder.parse(query);
assertTrue(lp.size()>0);
Operator op = lp.getSources().get(0);
-
+
assertTrue(op instanceof LOLoad);
LOLoad load = (LOLoad)op;
-
+
String p = load.getFileSpec().getFileName();
System.err.println("DEBUG: p:" + p + " expected:" + expected +", exectype:" + pc.getExecType());
if(noConversionExpected) {
assertEquals(expected, p);
} else {
- String protocol = pc.getExecType() == ExecType.MAPREDUCE ? "hdfs" : "file";
- // regex : A word character, i.e. [a-zA-Z_0-9] or '-' followed by ':' then any characters
+ String protocol = pc.getExecType() == cluster.getExecType() ? "hdfs" : "file";
+ // regex : A word character, i.e. [a-zA-Z_0-9] or '-' followed by ':' then any characters
String regex = "[\\-\\w:\\.]";
assertTrue(p.matches(".*" + protocol + "://" + regex + "*.*"));
assertEquals(expected, p.replaceAll(protocol + "://" + regex + "*/", "/"));
Modified: pig/trunk/test/org/apache/pig/test/TestLocalRearrange.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLocalRearrange.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLocalRearrange.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLocalRearrange.java Fri May 30 19:07:23 2014
@@ -17,7 +17,8 @@
*/
package org.apache.pig.test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileWriter;
@@ -28,9 +29,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
-import junit.framework.Assert;
-
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -50,33 +48,34 @@ import org.apache.pig.test.utils.GenRand
import org.apache.pig.test.utils.TestHelper;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
* Tests localrearrange db for
- * group db by $0
+ * group db by $0
*
*/
public class TestLocalRearrange {
-
+
POLocalRearrange lr;
Tuple t;
DataBag db;
- private static final MiniCluster cluster = MiniCluster.buildCluster();
+ private static final MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+
-
@Before
public void setUp() throws Exception {
Random r = new Random();
db = GenRandomData.genRandSmallTupDataBag(r, 10, 100);
}
-
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
private void setUp1() throws PlanException, ExecException{
lr = GenPhyOp.topLocalRearrangeOPWithPlanPlain(0,0,db.iterator().next());
POProject proj = GenPhyOp.exprProject();
@@ -106,32 +105,32 @@ public class TestLocalRearrange {
// The input data has 2 columns of which the first
// is the key
// With the optimized LocalRearrange, the part
- // of the "value" present in the "key" is
+ // of the "value" present in the "key" is
// excluded from the "value". So to reconstruct
// the true "value", create a tuple with "key" in
// first position and the "value" (val) we currently
// have in the second position
assertEquals(1, val.size());
-
+
Tuple actualVal = new DefaultTuple();
actualVal.append(key);
actualVal.append(val.get(0));
//Check if the index is same as input index
assertEquals((byte)0, (byte)(Byte)t.get(0));
-
+
//Check if the input bag contains the value tuple
assertTrue(TestHelper.bagContains(db, actualVal));
-
+
//Check if the input key and the output key are same
String inpKey = (String)actualVal.get(0);
assertEquals(0, inpKey.compareTo((String)t.get(1)));
++size;
}
-
+
//check if all the tuples in the input are generated
assertEquals(db.size(), size);
}
-
+
private void setUp2() throws PlanException, ExecException{
lr = GenPhyOp.topLocalRearrangeOPWithPlanPlain(0,0,db.iterator().next());
List<PhysicalPlan> plans = lr.getPlans();
@@ -139,7 +138,7 @@ public class TestLocalRearrange {
List<PhysicalPlan> plansT = lrT.getPlans();
plans.add(plansT.get(0));
lr.setPlans(plans);
-
+
POProject proj = GenPhyOp.exprProject();
proj.setColumn(0);
proj.setResultType(DataType.TUPLE);
@@ -151,7 +150,7 @@ public class TestLocalRearrange {
inputs.add(proj);
lr.setInputs(inputs);
}
-
+
@Test
public void testGetNextTuple2() throws ExecException, PlanException {
setUp2();
@@ -160,32 +159,32 @@ public class TestLocalRearrange {
Tuple t = (Tuple)res.result;
Tuple key = (Tuple)t.get(1);
Tuple val = (Tuple)t.get(2);
-
+
// The input data has 2 columns of which both
// are the key.
// With the optimized LocalRearrange, the part
- // of the "value" present in the "key" is
- // excluded from the "value". So in this case,
+ // of the "value" present in the "key" is
+ // excluded from the "value". So in this case,
// the "value" coming out of the LocalRearrange
// would be an empty tuple
assertEquals(0, val.size());
-
+
//Check if the index is same as input index
assertEquals((byte)0, (byte)(Byte)t.get(0));
-
+
// reconstruct value from tuple
val = key;
//Check if the input baf contains the value tuple
assertTrue(TestHelper.bagContains(db, val));
-
+
//Check if the input key and the output key are same
- Tuple inpKey = TupleFactory.getInstance().newTuple(2);
+ Tuple inpKey = TupleFactory.getInstance().newTuple(2);
inpKey.set(0, val.get(0));
inpKey.set(1, val.get(1));
- assertEquals(0, inpKey.compareTo((Tuple)t.get(1)));
+ assertEquals(inpKey, (Tuple)t.get(1));
++size;
}
-
+
//check if all the tuples in the input are generated
assertEquals(db.size(), size);
}
@@ -193,11 +192,11 @@ public class TestLocalRearrange {
@Test
public void testMultiQueryJiraPig1194() {
- // test case: POLocalRearrange doesn't handle nulls returned by POBinCond
-
+ // test case: POLocalRearrange doesn't handle nulls returned by POBinCond
+
String INPUT_FILE = "data.txt";
-
-
+
+
try {
PrintWriter w = new PrintWriter(new FileWriter(INPUT_FILE));
w.println("10\t2\t3");
@@ -210,25 +209,22 @@ public class TestLocalRearrange {
w.close();
Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
- PigServer myPig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ Util.resetStateForExecModeSwitch();
+ PigServer myPig = new PigServer(cluster.getExecType(), cluster.getProperties());
myPig.registerQuery("data = load '" + INPUT_FILE + "' as (a0, a1, a2);");
myPig.registerQuery("grp = GROUP data BY (((double) a2)/((double) a1) > .001 OR a0 < 11 ? a0 : 0);");
myPig.registerQuery("res = FOREACH grp GENERATE group, SUM(data.a1), SUM(data.a2);");
-
+
List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
- new String[] {
+ new String[] {
"(0,7000.0,5.0)",
- "(10,6.0,8.0)",
+ "(10,6.0,8.0)",
"(null,12.0,null)"
});
-
+
Iterator<Tuple> iter = myPig.openIterator("res");
- int counter = 0;
- while (iter.hasNext()) {
- assertEquals(expectedResults.get(counter++).toString(), iter.next().toString());
- }
- assertEquals(expectedResults.size(), counter);
+ Util.checkQueryOutputsAfterSort(iter, expectedResults);
} catch (Exception e) {
e.printStackTrace();
@@ -243,5 +239,5 @@ public class TestLocalRearrange {
}
}
}
-
+
}
Modified: pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Fri May 30 19:07:23 2014
@@ -1136,20 +1136,11 @@ public class TestMRCompiler {
System.out.println("Golden");
System.out.println("<<<" + goldenPlan + ">>>");
System.out.println("-------------");
-
+
String goldenPlanClean = Util.standardizeNewline(goldenPlan);
String compiledPlanClean = Util.standardizeNewline(compiledPlan);
- assertEquals(TestHelper.sortUDFs(removeSignature(goldenPlanClean)), TestHelper.sortUDFs(removeSignature(compiledPlanClean)));
- }
-
- /**
- * this removes the signature from the serialized plan
- * changing the way the unique signature is generated should not break this test
- * @param plan the plan to canonicalize
- * @return the cleaned up plan
- */
- private String removeSignature(String plan) {
- return plan.replaceAll("','','[^']*','scope','true'\\)\\)", "','','','scope','true'))");
+ assertEquals(TestHelper.sortUDFs(Util.removeSignature(goldenPlanClean)),
+ TestHelper.sortUDFs(Util.removeSignature(compiledPlanClean)));
}
public static class TestCollectableLoadFunc extends PigStorage implements CollectableLoadFunc {
@@ -1223,11 +1214,11 @@ public class TestMRCompiler {
Utils.getSchemaFromString("a : int,b :float ,c : int")
);
}
-
+
//PIG-2146
@Test
public void testStorerLimit() throws Exception {
- // test if the POStore in the 1st mr plan
+ // test if the POStore in the 1st mr plan
// use the right StoreFunc
String query = "a = load 'input1';" +
"b = limit a 10;" +
@@ -1235,13 +1226,14 @@ public class TestMRCompiler {
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
-
+
LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
la.visit();
la.adjust();
-
+
MapReduceOper firstMrOper = mrPlan.getRoots().get(0);
POStore store = (POStore)firstMrOper.reducePlan.getLeaves().get(0);
assertEquals(store.getStoreFunc().getClass().getName(), "org.apache.pig.impl.io.InterStorage");
}
-}
\ No newline at end of file
+}
+
Modified: pig/trunk/test/org/apache/pig/test/TestMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMapReduce.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMapReduce.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMapReduce.java Fri May 30 19:07:23 2014
@@ -29,13 +29,12 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Iterator;
-import java.util.Properties;
import java.util.Map.Entry;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.EvalFunc;
-import org.apache.pig.ExecType;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigServer;
import org.apache.pig.backend.datastorage.ElementDescriptor;
@@ -56,21 +55,21 @@ import org.junit.Test;
public class TestMapReduce {
private Log log = LogFactory.getLog(getClass());
-
- static MiniCluster cluster = MiniCluster.buildCluster();
+
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pig;
-
+
@Before
public void setUp() throws Exception {
- pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pig = new PigServer(cluster.getExecType(), cluster.getProperties());
}
-
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
@Test
public void testBigGroupAll() throws Throwable {
@@ -112,19 +111,19 @@ public class TestMapReduce {
/**
* This test checks records that align perfectly on
- * bzip block boundaries and hdfs block boundaries
+ * bzip block boundaries and hdfs block boundaries
*/
@Test
public void testBZip2Aligned() throws Throwable {
int offsets[] = { 219642, 219643, 219644, 552019, 552020 };
for(int i = 1; i < offsets.length; i ++) {
-
+
Properties props = new Properties();
for (Entry<Object, Object> entry : cluster.getProperties().entrySet()) {
props.put(entry.getKey(), entry.getValue());
}
props.setProperty("mapred.max.split.size", Integer.toString(offsets[i]));
- PigContext pigContext = new PigContext(ExecType.MAPREDUCE, props);
+ PigContext pigContext = new PigContext(cluster.getExecType(), props);
PigServer pig = new PigServer(pigContext);
pig.registerQuery("a = load '"
+ Util.generateURI(
@@ -143,7 +142,7 @@ public class TestMapReduce {
//assertEquals("1000000", it.next().getField(0));
}
}
-
+
public Double bigGroupAll( File tmpFile ) throws Throwable {
String query = "foreach (group (load '"
@@ -157,7 +156,7 @@ public class TestMapReduce {
return DataType.toDouble(t.get(0));
}
-
+
static public class MyApply extends EvalFunc<DataBag> {
String field0 = "Got";
public MyApply() {}
@@ -210,14 +209,14 @@ public class TestMapReduce {
public void setNulls(boolean hasNulls ) { this.hasNulls=hasNulls; }
/**
- *
+ *
*/
public MyStorage() {
// initialize delimiter to be "-" for output
// since that is the delimiter in the tests below
super("-");
}
-
+
@Override
public Tuple getNext() throws IOException {
if (count < COUNT) {
@@ -289,7 +288,7 @@ public class TestMapReduce {
String[][] data = genDataSetFile1( 10, true );
storeFunction( data);
}
-
+
public void storeFunction(String[][] data) throws Throwable {
File tmpFile=TestHelper.createTempFile(data) ;
@@ -362,7 +361,7 @@ public class TestMapReduce {
assertEquals( MyStorage.COUNT, count );
tmpFile.delete();
}
-
+
@Test
public void testQualifiedFunctionsWithNulls() throws Throwable {
@@ -401,7 +400,7 @@ public class TestMapReduce {
assertEquals( MyStorage.COUNT, count );
tmpFile.delete();
}
-
+
@Test
public void testDefinedFunctions() throws Throwable {
@@ -489,7 +488,7 @@ public class TestMapReduce {
@Test
public void testPigServer() throws Throwable {
log.debug("creating pig server");
- PigContext pigContext = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
+ PigContext pigContext = new PigContext(cluster.getExecType(), cluster.getProperties());
PigServer pig = new PigServer(pigContext);
System.out.println("testing capacity");
long capacity = pig.capacity();
@@ -523,11 +522,11 @@ public class TestMapReduce {
* 3 3
* 4 4
* 5 5
- * 6
+ * 6
* 7 7
- *
+ *
* 9 9
- *
+ *
*/
private String[][] genDataSetFile1( int dataLength, boolean hasNulls ) throws IOException {
@@ -544,7 +543,7 @@ public class TestMapReduce {
data[i][1] = new Integer(i).toString();
} else if ( i == 6 ) {
-
+
data[i][0] = new Integer(i).toString();
data[i][1] = "";
Modified: pig/trunk/test/org/apache/pig/test/TestMapReduce2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMapReduce2.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMapReduce2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMapReduce2.java Fri May 30 19:07:23 2014
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Iterator;
-import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
import org.apache.pig.test.utils.TestHelper;
@@ -34,12 +33,12 @@ import org.junit.Test;
public class TestMapReduce2 {
- static MiniCluster cluster = MiniCluster.buildCluster();
+ static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
private PigServer pig ;
public TestMapReduce2() throws Throwable {
- pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()) ;
+ pig = new PigServer(cluster.getExecType(), cluster.getProperties()) ;
}
@AfterClass
Modified: pig/trunk/test/org/apache/pig/test/TestMultiQuery.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQuery.java Fri May 30 19:07:23 2014
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.data.Tuple;
@@ -51,7 +52,7 @@ public class TestMultiQuery {
Util.copyFromLocalToLocal(
"test/org/apache/pig/test/data/passwd2", "passwd2");
Properties props = new Properties();
- props.setProperty("opt.multiquery", ""+true);
+ props.setProperty(PigConfiguration.OPT_MULTIQUERY, ""+true);
myPig = new PigServer(ExecType.LOCAL, props);
}
Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java?rev=1598702&r1=1598701&r2=1598702&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java Fri May 30 19:07:23 2014
@@ -29,8 +29,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import junit.framework.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +40,7 @@ import org.apache.hadoop.mapreduce.Outpu
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.pig.ExecType;
import org.apache.pig.LoadFunc;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceSchema;
import org.apache.pig.StoreFunc;
@@ -69,17 +68,17 @@ public class TestMultiQueryBasic {
Util.copyFromLocalToLocal(
"test/org/apache/pig/test/data/passwd2", "passwd2");
Properties props = new Properties();
- props.setProperty("opt.multiquery", ""+true);
+ props.setProperty(PigConfiguration.OPT_MULTIQUERY, ""+true);
myPig = new PigServer(ExecType.LOCAL, props);
}
-
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "passwd");
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "passwd2");
deleteOutputFiles();
}
-
+
@Before
public void setUp() throws Exception {
deleteOutputFiles();
@@ -87,10 +86,10 @@ public class TestMultiQueryBasic {
@After
public void tearDown() throws Exception {
-
+
}
-
-
+
+
@Test
public void testMultiQueryWithTwoStores2() throws Exception {
@@ -133,8 +132,8 @@ public class TestMultiQueryBasic {
myPig.executeBatch();
myPig.discardBatch();
- }
-
+ }
+
@Test
public void testMultiQueryPhase3BaseCase2() throws Exception {
@@ -155,16 +154,16 @@ public class TestMultiQueryBasic {
myPig.registerQuery("c2 = foreach c1 generate group, SUM(c.uid);");
myPig.registerQuery("store c2 into 'output2';");
myPig.registerQuery("d1 = group d by gid;");
- myPig.registerQuery("d2 = foreach d1 generate group, AVG(d.uid);");
+ myPig.registerQuery("d2 = foreach d1 generate group, AVG(d.uid);");
myPig.registerQuery("store d2 into 'output3';");
-
+
List<ExecJob> jobs = myPig.executeBatch();
-
+
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
-
+ }
+
@Test
public void testMultiQueryPhase3WithoutCombiner2() throws Exception {
@@ -184,17 +183,17 @@ public class TestMultiQueryBasic {
myPig.registerQuery("c1 = group c by gid;");
myPig.registerQuery("c2 = foreach c1 generate group, SUM(c.uid) - COUNT(c.uid);");
myPig.registerQuery("store c2 into 'output2';");
- myPig.registerQuery("d1 = group d by gid;");
+ myPig.registerQuery("d1 = group d by gid;");
myPig.registerQuery("d2 = foreach d1 generate group, MAX(d.uid) - MIN(d.uid);");
myPig.registerQuery("store d2 into 'output3';");
-
+
List<ExecJob> jobs = myPig.executeBatch();
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
-
+ }
+
@Test
public void testMultiQueryPhase3WithMixedCombiner2() throws Exception {
@@ -214,17 +213,17 @@ public class TestMultiQueryBasic {
myPig.registerQuery("c1 = group c by gid;");
myPig.registerQuery("c2 = foreach c1 generate group, SUM(c.uid);");
myPig.registerQuery("store c2 into 'output2';");
- myPig.registerQuery("d1 = group d by gid;");
+ myPig.registerQuery("d1 = group d by gid;");
myPig.registerQuery("d2 = foreach d1 generate group, MAX(d.uid) - MIN(d.uid);");
myPig.registerQuery("store d2 into 'output3';");
-
+
List<ExecJob> jobs = myPig.executeBatch();
assertEquals(3, jobs.size());
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
+ }
@Test
public void testMultiQueryPhase3WithDifferentMapDataTypes2() throws Exception {
@@ -248,15 +247,15 @@ public class TestMultiQueryBasic {
myPig.registerQuery("d1 = group d by $1;");
myPig.registerQuery("d2 = foreach d1 generate group, COUNT(d.uid);");
myPig.registerQuery("store d2 into 'output3';");
-
+
List<ExecJob> jobs = myPig.executeBatch();
assertEquals(3, jobs.size());
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
-
+ }
+
@Test
public void testMultiQueryPhase3WithDifferentMapDataTypes3() throws Exception {
@@ -278,14 +277,14 @@ public class TestMultiQueryBasic {
myPig.registerQuery("d = group a by (name, gpa);");
myPig.registerQuery("e = foreach d generate flatten(group), MIN(a.age);");
myPig.registerQuery("store e into 'output2';");
-
+
myPig.executeBatch();
-
+
myPig.registerQuery("a = load 'output1' as (grp:chararray, cnt:long) ;");
Iterator<Tuple> it = myPig.openIterator("a");
assertEquals(Util.getPigConstant("('all', 5l)"), it.next());
assertFalse(it.hasNext());
-
+
myPig.registerQuery("a = load 'output2' as (name:chararray, gpa:double, age:int);");
it = myPig.openIterator("a");
int i = 0;
@@ -299,8 +298,8 @@ public class TestMultiQueryBasic {
assertEquals(expectedResults.get(t.get(0)), t);
}
assertEquals(3, i);
- }
-
+ }
+
@Test
public void testMultiQueryPhase3StreamingInReducer2() throws Exception {
@@ -317,19 +316,19 @@ public class TestMultiQueryBasic {
myPig.registerQuery("store D into 'output1';");
myPig.registerQuery("E = group A4 by $2;");
myPig.registerQuery("F = foreach E generate group, COUNT(A4);");
- myPig.registerQuery("store F into 'output2';");
+ myPig.registerQuery("store F into 'output2';");
myPig.registerQuery("G = group A1 by $2;");
- myPig.registerQuery("H = foreach G generate group, COUNT(A1);");
+ myPig.registerQuery("H = foreach G generate group, COUNT(A1);");
myPig.registerQuery("store H into 'output3';");
-
+
List<ExecJob> jobs = myPig.executeBatch();
assertEquals(3, jobs.size());
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
-
+ }
+
@Test
public void testMultiQueryWithPigMixL12_2() throws Exception {
@@ -340,7 +339,7 @@ public class TestMultiQueryBasic {
myPig.registerQuery("a = load 'passwd' " +
"using PigStorage(':') as (uname, passwd, uid, gid);");
myPig.registerQuery("b = foreach a generate uname, passwd, uid, gid;");
- myPig.registerQuery("split b into c1 if uid > 5, c2 if uid <= 5 ;");
+ myPig.registerQuery("split b into c1 if uid > 5, c2 if uid <= 5 ;");
myPig.registerQuery("split c1 into d1 if gid < 5, d2 if gid >= 5;");
myPig.registerQuery("e = group d1 by uname;");
myPig.registerQuery("e1 = foreach e generate group, MAX(d1.uid);");
@@ -359,18 +358,18 @@ public class TestMultiQueryBasic {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
}
-
+
@Test
public void testMultiQueryWithCoGroup_2() throws Exception {
System.out.println("===== multi-query with CoGroup (2) =====");
-
+
myPig.setBatchOn();
myPig.registerQuery("a = load 'passwd' " +
"using PigStorage(':') as (uname, passwd, uid, gid);");
myPig.registerQuery("store a into 'output1' using BinStorage();");
- myPig.registerQuery("b = load 'output1' using BinStorage() as (uname, passwd, uid, gid);");
+ myPig.registerQuery("b = load 'output1' using BinStorage() as (uname, passwd, uid, gid);");
myPig.registerQuery("c = load 'passwd2' " +
"using PigStorage(':') as (uname, passwd, uid, gid);");
myPig.registerQuery("d = cogroup b by (uname, uid) inner, c by (uname, uid) inner;");
@@ -384,7 +383,7 @@ public class TestMultiQueryBasic {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
}
-
+
@Test
public void testMultiQueryWithFJ_2() throws Exception {
@@ -409,15 +408,15 @@ public class TestMultiQueryBasic {
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
-
+ }
+
@Test
public void testMultiQueryWithIntermediateStores_2() throws Exception {
System.out.println("===== multi-query with intermediate stores (2) =====");
myPig.setBatchOn();
-
+
myPig.registerQuery("a = load 'passwd' " +
"using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int, gid:int);");
myPig.registerQuery("store a into 'output1';");
@@ -430,7 +429,7 @@ public class TestMultiQueryBasic {
for (ExecJob job : jobs) {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
- }
+ }
@Test
public void testMultiQueryWithSplitInMapAndMultiMerge() throws Exception {
@@ -438,9 +437,9 @@ public class TestMultiQueryBasic {
// clean up any existing dirs/files
String[] toClean = {"tmwsimam-input.txt", "foo1", "foo2", "foo3", "foo4" };
for (int j = 0; j < toClean.length; j++) {
- Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), toClean[j]);
+ Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), toClean[j]);
}
-
+
// the data below is tab delimited
String[] inputData = {
"1 a b e f i j m n",
@@ -448,21 +447,21 @@ public class TestMultiQueryBasic {
"3 c d g h k l o p",
"4 c d g h k l o p" };
Util.createLocalInputFile("tmwsimam-input.txt", inputData);
- String query =
+ String query =
"A = LOAD 'tmwsimam-input.txt' " +
"as (f0:chararray, f1:chararray, f2:chararray, f3:chararray, " +
"f4:chararray, f5:chararray, f6:chararray, f7:chararray, f8:chararray); " +
"B = FOREACH A GENERATE f0, f1, f2, f3, f4;" +
- "B1 = foreach B generate f0, f1, f2;" +
- "C = GROUP B1 BY (f1, f2);" +
+ "B1 = foreach B generate f0, f1, f2;" +
+ "C = GROUP B1 BY (f1, f2);" +
"STORE C into 'foo1' using BinStorage();" +
- "B2 = FOREACH B GENERATE f0, f3, f4;" +
+ "B2 = FOREACH B GENERATE f0, f3, f4;" +
"E = GROUP B2 BY (f3, f4);" +
"STORE E into 'foo2' using BinStorage();" +
"F = FOREACH A GENERATE f0, f5, f6, f7, f8;" +
"F1 = FOREACH F GENERATE f0, f5, f6;" +
"G = GROUP F1 BY (f5, f6);" +
- "STORE G into 'foo3' using BinStorage();" +
+ "STORE G into 'foo3' using BinStorage();" +
"F2 = FOREACH F GENERATE f0, f7, f8;" +
"I = GROUP F2 BY (f7, f8);" +
"STORE I into 'foo4' using BinStorage();" +
@@ -470,25 +469,25 @@ public class TestMultiQueryBasic {
myPig.setBatchOn();
Util.registerMultiLineQuery(myPig, query);
myPig.executeBatch();
-
+
String templateLoad = "a = load 'foo' using BinStorage();";
-
+
Map<Tuple, DataBag> expectedResults = new HashMap<Tuple, DataBag>();
- expectedResults.put((Tuple)Util.getPigConstant("('a','b')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('a','b')"),
(DataBag)Util.getPigConstant("{('1','a','b'),('2','a','b')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('c','d')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('c','d')"),
(DataBag)Util.getPigConstant("{('3','c','d'),('4','c','d')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('e','f')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('e','f')"),
(DataBag)Util.getPigConstant("{('1','e','f'),('2','e','f')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('g','h')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('g','h')"),
(DataBag)Util.getPigConstant("{('3','g','h'),('4','g','h')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('i','j')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('i','j')"),
(DataBag)Util.getPigConstant("{('1','i','j'),('2','i','j')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('k','l')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('k','l')"),
(DataBag)Util.getPigConstant("{('3','k','l'),('4','k','l')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('m','n')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('m','n')"),
(DataBag)Util.getPigConstant("{('1','m','n'),('2','m','n')}"));
- expectedResults.put((Tuple)Util.getPigConstant("('o','p')"),
+ expectedResults.put((Tuple)Util.getPigConstant("('o','p')"),
(DataBag)Util.getPigConstant("{('3','o','p'),('4','o','p')}"));
String[] outputDirs = { "foo1", "foo2", "foo3", "foo4" };
for(int k = 0; k < outputDirs.length; k++) {
@@ -504,11 +503,11 @@ public class TestMultiQueryBasic {
}
// cleanup
for (int j = 0; j < toClean.length; j++) {
- Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), toClean[j]);
+ Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), toClean[j]);
}
-
+
}
-
+
@Test
public void testMultiQueryWithTwoStores2Execs() throws Exception {
@@ -548,11 +547,11 @@ public class TestMultiQueryBasic {
myPig.executeBatch();
myPig.discardBatch();
}
-
+
/**
* Test that pig calls checkOutputSpecs() method of the OutputFormat (if the
- * StoreFunc defines an OutputFormat as the return value of
- * {@link StoreFunc#getStorePreparationClass()}
+ * StoreFunc defines an OutputFormat as the return value of
+ * {@link StoreFunc#getStorePreparationClass()}
* @throws IOException
*/
@Test
@@ -567,13 +566,13 @@ public class TestMultiQueryBasic {
myPig.setBatchOn();
Util.registerMultiLineQuery(myPig, query);
myPig.executeBatch();
-
+
// check that files were created as a result of the
// checkOutputSpecs() method of the OutputFormat being called
FileSystem fs = FileSystem.getLocal(new Configuration());
assertEquals(true, fs.exists(new Path("output1_checkOutputSpec_test")));
assertEquals(true, fs.exists(new Path("output2_checkOutputSpec_test")));
-
+
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "output1_checkOutputSpec_test");
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "output2_checkOutputSpec_test");
}
@@ -615,23 +614,23 @@ public class TestMultiQueryBasic {
}
assertEquals(3, i);
}
-
+
private static final String DUMMY_STORE_WITH_OUTPUTFORMAT_CLASS
= "org.apache.pig.test.TestMultiQueryBasic\\$DummyStoreWithOutputFormat";
public static class DummyStoreWithOutputFormat extends StoreFunc {
-
+
public DummyStoreWithOutputFormat() {
}
@Override
public void putNext(Tuple f) throws IOException {
-
+
}
@Override
public void checkSchema(ResourceSchema s) throws IOException {
-
+
}
@Override
@@ -644,7 +643,7 @@ public class TestMultiQueryBasic {
public void prepareToWrite(
org.apache.hadoop.mapreduce.RecordWriter writer)
throws IOException {
-
+
}
@Override
@@ -658,21 +657,21 @@ public class TestMultiQueryBasic {
throws IOException {
Configuration conf = job.getConfiguration();
conf.set("mapred.output.dir", location);
-
+
}
-
+
@Override
public void setStoreFuncUDFContextSignature(String signature) {
}
-
+
}
-
+
@SuppressWarnings({ "unchecked" })
public static class DummyOutputFormat
extends OutputFormat<WritableComparable, Tuple> {
public DummyOutputFormat() {
-
+
}
@Override
public void checkOutputSpecs(JobContext context) throws IOException,
@@ -694,9 +693,9 @@ public class TestMultiQueryBasic {
InterruptedException {
return null;
}
-
+
}
-
+
// --------------------------------------------------------------------------
// Helper methods