You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zl...@apache.org on 2017/02/24 03:34:40 UTC
svn commit: r1784224 [13/17] - in /pig/branches/spark: ./ bin/ conf/
contrib/piggybank/java/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachel...
Modified: pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestMRJobStats.java Fri Feb 24 03:34:37 2017
@@ -102,7 +102,7 @@ public class TestMRJobStats {
try {
Constructor<MRJobStats> con = MRJobStats.class.getDeclaredConstructor(String.class, JobGraph.class);
con.setAccessible(true);
- MRJobStats jobStats = con.newInstance(name, plan);
+ MRJobStats jobStats = (MRJobStats) con.newInstance(name, plan);
return jobStats;
} catch (Exception e) {
return null;
@@ -202,49 +202,14 @@ public class TestMRJobStats {
}
}
- private POStore createPOStoreForFileBasedSystemWithSubDirectories(long size, StoreFuncInterface storeFunc, Configuration conf) throws Exception {
- File root = createTmpDirectory("outputRoot", null);
- File dir1 = createTmpDirectory("dir1", root);
- File dir2 = createTmpDirectory("dir2", root);
- createTmpFile("tempFile1", size, dir1);
- createTmpFile("tempFile2", size, dir2);
-
- storeFunc.setStoreLocation(root.getAbsolutePath(), new Job(conf));
- FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName());
- POStore poStore = new POStore(new OperatorKey());
- poStore.setSFile(new FileSpec(root.getAbsolutePath(), funcSpec));
- poStore.setStoreFunc(storeFunc);
- poStore.setUp();
-
- return poStore;
- }
-
- private static File createTmpDirectory(String name, File root) throws Exception {
- File directory = File.createTempFile(name, "", root);
-
- if (!(directory.delete())) {
- throw new IOException("Could not delete temp file: " + directory.getAbsolutePath());
- }
-
- if (!(directory.mkdir())) {
- throw new IOException("Could not create temp directory: " + directory.getAbsolutePath());
- }
-
- return directory;
- }
+ private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc,
+ Configuration conf) throws Exception {
- private static File createTmpFile(String name, long size, File directory) throws Exception {
- File file = directory == null ? File.createTempFile(name, ".tmp") : File.createTempFile(name, ".tmp", directory);
+ File file = File.createTempFile("tempFile", ".tmp");
file.deleteOnExit();
RandomAccessFile f = new RandomAccessFile(file, "rw");
f.setLength(size);
f.close();
- return file;
- }
-
- private static POStore createPOStoreForFileBasedSystem(long size, StoreFuncInterface storeFunc,
- Configuration conf) throws Exception {
- File file = createTmpFile("tempFile", size, null);
storeFunc.setStoreLocation(file.getAbsolutePath(), new Job(conf));
FuncSpec funcSpec = new FuncSpec(storeFunc.getClass().getCanonicalName());
@@ -271,7 +236,7 @@ public class TestMRJobStats {
}
@Test
- public void testGetOutputSizeUsingFileBasedStorage() throws Exception {
+ public void testGetOuputSizeUsingFileBasedStorage() throws Exception {
// By default, FileBasedOutputSizeReader is used to compute the size of output.
Configuration conf = new Configuration();
@@ -284,20 +249,7 @@ public class TestMRJobStats {
}
@Test
- public void testGetOutputSizeUsingFileBasedStorageWithSubDirectories() throws Exception {
- // By default, FileBasedOutputSizeReader is used to compute the size of output.
- Configuration conf = new Configuration();
-
- long size = 2L * 1024 * 1024 * 1024;
- long outputSize = JobStats.getOutputSize(
- createPOStoreForFileBasedSystemWithSubDirectories(size, new PigStorageWithStatistics(), conf), conf);
-
- assertEquals("The returned output size is expected to be sum of file sizes in the sub-directories",
- 2 * size, outputSize);
- }
-
- @Test
- public void testGetOutputSizeUsingNonFileBasedStorage1() throws Exception {
+ public void testGetOuputSizeUsingNonFileBasedStorage1() throws Exception {
// By default, FileBasedOutputSizeReader is used to compute the size of output.
Configuration conf = new Configuration();
@@ -311,7 +263,7 @@ public class TestMRJobStats {
}
@Test
- public void testGetOutputSizeUsingNonFileBasedStorage2() throws Exception {
+ public void testGetOuputSizeUsingNonFileBasedStorage2() throws Exception {
// Register a custom output size reader in configuration
Configuration conf = new Configuration();
conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
@@ -327,7 +279,7 @@ public class TestMRJobStats {
}
@Test(expected = RuntimeException.class)
- public void testGetOutputSizeUsingNonFileBasedStorage3() throws Exception {
+ public void testGetOuputSizeUsingNonFileBasedStorage3() throws Exception {
// Register an invalid output size reader in configuration, and verify
// that an exception is thrown at run-time.
Configuration conf = new Configuration();
@@ -340,7 +292,7 @@ public class TestMRJobStats {
}
@Test
- public void testGetOutputSizeUsingNonFileBasedStorage4() throws Exception {
+ public void testGetOuputSizeUsingNonFileBasedStorage4() throws Exception {
// Register a comma-separated list of readers in configuration, and
// verify that the one that supports a non-file-based uri is used.
Configuration conf = new Configuration();
@@ -358,7 +310,7 @@ public class TestMRJobStats {
}
@Test
- public void testGetOutputSizeUsingNonFileBasedStorage5() throws Exception {
+ public void testGetOuputSizeUsingNonFileBasedStorage5() throws Exception {
Configuration conf = new Configuration();
long size = 2L * 1024 * 1024 * 1024;
Modified: pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestMacroExpansion.java Fri Feb 24 03:34:37 2017
@@ -1187,33 +1187,6 @@ public class TestMacroExpansion {
validateFailure(sb.toString(), expectedErr, "at");
}
-
- @Test
- public void lineNumberTest3() throws Throwable {
- StringBuilder sb = new StringBuilder();
- sb.append("/*\n" +
- " * extra lines to offset the line number for the macro\n" +
- " *\n" +
- " *\n" +
- " */\n" +
- "\n" +
- "\n" +
- "define mymacro() returns void {\n" +
- "A = load 'x' as ( u:int, v:long, w:bytearray);\n" +
- "B = limit A 100;\n" +
- "C = filter_typo B by 2 > 1;\n" +
- "D = load 'y' as (d1, d2);\n" +
- "E = join C by ( $0, $1 ), D by ( d1, d2 ) using 'replicated' parallel 16;\n" +
- "F = store E into 'output';\n" +
- "};\n" +
- "mymacro();\n"
- );
-
- String expectedErr =
- "/myscript.pig, line 11, column 0> Syntax error, unexpected symbol at or near 'C'";
-
- validateFailure(sb.toString(), expectedErr, "/myscript.pig, line ");
- }
//see Pig-2184
@Test
@@ -2279,135 +2252,6 @@ public class TestMacroExpansion {
verify(script, expected);
}
-
- // When declare-in-macro, macro param and command-line param contain the
- // same name, last declare wins
- @Test
- public void testParamOverLap1() throws Exception {
- String macro =
- "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
- " %declare number 333;\n" +
- " $G = GROUP $REL by $key parallel $number;\n" +
- "};";
- createFile("my_macro.pig", macro);
-
- String script =
- "%declare number 111;\n" +
- "IMPORT 'my_macro.pig';\n" +
- "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
- "result = mygroupby(data, i, 222);\n" +
- "STORE result INTO 'test.out' USING PigStorage();";
-
- String expected =
- "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
- "result = GROUP data by (i) parallel 333;\n" +
- "STORE result INTO 'test.out' USING PigStorage();\n";
-
- verify(script, expected);
- }
-
- // When default-in-macro, macro param and command-line param contain the
- // same name, then default should be ignored and macro param to be taken
- @Test
- public void testParamOverLap2() throws Exception {
- String macro =
- "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
- " %default number 333;\n" +
- " $G = GROUP $REL by $key parallel $number;\n" +
- "};";
- createFile("my_macro.pig", macro);
-
- String script =
- "%declare number 111;\n" +
- "IMPORT 'my_macro.pig';\n" +
- "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
- "result = mygroupby(data, i, 222);\n" +
- "STORE result INTO 'test.out' USING PigStorage();";
-
- String expected =
- "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
- "result = GROUP data by (i) parallel 222;\n" +
- "STORE result INTO 'test.out' USING PigStorage();\n";
-
- verify(script, expected);
- }
-
- // Overlapping of macro param and command-line param used to be disallowed.
- // Now, simply taking the macro param when this happens
- @Test
- public void testParamOverLap3() throws Exception {
- String macro =
- "DEFINE mygroupby(REL, key, number) RETURNS G {\n" +
- " $G = GROUP $REL by $key parallel $number;\n" +
- "};";
- createFile("my_macro.pig", macro);
-
- String script =
- "%default number 111;\n" +
- "IMPORT 'my_macro.pig';\n" +
- "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
- "result = mygroupby(data, i, 222);\n" +
- "STORE result INTO 'test.out' USING PigStorage();";
-
- String expected =
- "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
- "result = GROUP data by (i) parallel 222;\n" +
- "STORE result INTO 'test.out' USING PigStorage();\n";
-
- verify(script, expected);
- }
-
- // Testing inline declare and commandline param overlap.
- // testParamOverLap1 should cover this case as well but creating a specific
- // case since this pair used to fail with NPE
- @Test
- public void testParamOverLap4() throws Exception {
- String macro =
- "DEFINE mygroupby(REL, key) RETURNS G {\n" +
- " %declare number 333;\n" +
- " $G = GROUP $REL by $key parallel $number;\n" +
- "};";
- createFile("my_macro.pig", macro);
-
- String script =
- "%default number 111;\n" +
- "IMPORT 'my_macro.pig';\n" +
- "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
- "result = mygroupby(data, i);\n" +
- "STORE result INTO 'test.out' USING PigStorage();";
-
- String expected =
- "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
- "result = GROUP data by (i) parallel 333;\n" +
- "STORE result INTO 'test.out' USING PigStorage();\n";
-
- verify(script, expected);
- }
-
- // default-in-macro should yield to command-line param
- @Test
- public void testParamOverLap5() throws Exception {
- String macro =
- "DEFINE mygroupby(REL, key) RETURNS G {\n" +
- " %default number 333;\n" +
- " $G = GROUP $REL by $key parallel $number;\n" +
- "};";
- createFile("my_macro.pig", macro);
-
- String script =
- "%declare number 111;\n" +
- "IMPORT 'my_macro.pig';\n" +
- "data = LOAD '1234.txt' USING PigStorage() AS (i: int);\n" +
- "result = mygroupby(data, i);\n" +
- "STORE result INTO 'test.out' USING PigStorage();";
-
- String expected =
- "data = LOAD '1234.txt' USING PigStorage() AS i:int;\n" +
- "result = GROUP data by (i) parallel 111;\n" +
- "STORE result INTO 'test.out' USING PigStorage();\n";
-
- verify(script, expected);
- }
//-------------------------------------------------------------------------
Modified: pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestMultiQuery.java Fri Feb 24 03:34:37 2017
@@ -883,49 +883,6 @@ public class TestMultiQuery {
}
@Test
- public void testMultiQueryJiraPig4883() throws Exception {
- Storage.Data data = Storage.resetData(myPig);
- data.set("inputLocation",
- Storage.tuple("c", "12"), Storage.tuple("d", "-12"));
- myPig.setBatchOn();
- myPig.registerQuery("A = load 'inputLocation' using mock.Storage();");
- myPig.registerQuery("A = foreach A generate (chararray)$0 as id, (long)$1 as val;");
- myPig.registerQuery("B = filter A by val > 0;");
- myPig.registerQuery("B1 = group B by val;");
- myPig.registerQuery("B1 = foreach B1 generate group as name, COUNT(B) as value;");
- myPig.registerQuery("B1 = foreach B1 generate (chararray)name,value;");
- myPig.registerQuery("store B1 into 'output1' using mock.Storage();");
- myPig.registerQuery("B2 = group B by id;");
- myPig.registerQuery("B2 = foreach B2 generate group as name, COUNT(B) as value;");
- myPig.registerQuery("store B2 into 'output2' using mock.Storage();");
- myPig.registerQuery("C = filter A by val < 0;");
- myPig.registerQuery("C1 = group C by val;");
- myPig.registerQuery("C1 = foreach C1 generate group as name, COUNT(C) as value;");
- myPig.registerQuery("store C1 into 'output3' using mock.Storage();");
- myPig.registerQuery("C2 = group C by id;");
- myPig.registerQuery("C2 = foreach C2 generate group as name, COUNT(C) as value;");
- myPig.registerQuery("store C2 into 'output4' using mock.Storage();");
- myPig.executeBatch();
-
- List<Tuple> actualResults = data.get("output1");
- String[] expectedResults = new String[]{"(12, 1)"};
- Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("B1")));
-
-
- actualResults = data.get("output2");
- expectedResults = new String[]{"(c,1)"};
- Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("B2")));
-
- actualResults = data.get("output3");
- expectedResults = new String[]{"(-12, 1)"};
- Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("C1")));
-
- actualResults = data.get("output4");
- expectedResults = new String[]{"(d,1)"};
- Util.checkQueryOutputsAfterSortRecursive(actualResults.iterator(), expectedResults, org.apache.pig.newplan.logical.Util.translateSchema(myPig.dumpSchema("C2")));
- }
-
- @Test
public void testMultiQueryJiraPig4899() throws Exception {
myPig.setBatchOn();
Modified: pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestMultiQueryCompiler.java Fri Feb 24 03:34:37 2017
@@ -1558,7 +1558,14 @@ public class TestMultiQueryCompiler {
MROperPlan mrp = null;
try {
- mrp = launcher.compile(pp, myPig.getPigContext());
+ java.lang.reflect.Method compile = launcher.getClass()
+ .getDeclaredMethod("compile",
+ new Class[] { PhysicalPlan.class, PigContext.class });
+
+ compile.setAccessible(true);
+
+ mrp = (MROperPlan) compile.invoke(launcher, new Object[] { pp, myPig.getPigContext() });
+
Assert.assertNotNull(mrp);
} catch (Exception e) {
Modified: pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java Fri Feb 24 03:34:37 2017
@@ -57,8 +57,6 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.IsNullExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
import org.apache.pig.newplan.logical.expression.MapLookupExpression;
-import org.apache.pig.newplan.logical.expression.NotEqualExpression;
-import org.apache.pig.newplan.logical.expression.NotExpression;
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
@@ -129,17 +127,6 @@ public class TestNewPartitionFilterPushD
* @throws Exception
*/
@Test
- public void testPartIsNullFilter() throws Exception {
- String q = query + "b = filter a by srcid is null;" + "store b into 'out';";
- test(q, Arrays.asList("srcid"),
- null, "(srcid is null)");
- }
-
- /**
- * test case where filter only contains condition on partition cols
- * @throws Exception
- */
- @Test
public void testOnlyPartFilter2() throws Exception {
String q = query + "b = filter a by mrkt == 'us';" + "store b into 'out';";
test(q, Arrays.asList("srcid", "mrkt"),
@@ -698,15 +685,6 @@ public class TestNewPartitionFilterPushD
testFull(q, "((srcid < 5) or (srcid == 10))", "((f1 < 5) or (f2 == 'UK'))", false);
}
- // PIG-4940
- @Test
- public void testUnaryExpressions() throws Exception {
- String q = query + "b = filter a by srcid == 10 and not browser#'type' is null;" +
- "store b into 'out';";
- test(q, Arrays.asList("srcid"), "(srcid == 10)",
- "(not (browser#'type' is null))", true);
- }
-
//// helper methods ///////
private PartitionFilterExtractor test(String query, List<String> partitionCols,
String expPartFilterString, String expFilterString)
@@ -871,7 +849,7 @@ public class TestNewPartitionFilterPushD
return "(" + input + ")";
}
- public static String getTestExpression(LogicalExpression op) throws FrontendException {
+ private static String getTestExpression(LogicalExpression op) throws FrontendException {
if(op == null) {
return null;
}
@@ -893,8 +871,6 @@ public class TestNewPartitionFilterPushD
opStr = " and ";
} else if (op instanceof OrExpression) {
opStr = " or ";
- } else if (op instanceof NotEqualExpression) {
- opStr = " != ";
} else {
opStr = op.getName();
}
@@ -914,9 +890,6 @@ public class TestNewPartitionFilterPushD
int colind = ((DereferenceExpression) op).getBagColumns().get(0);
String column = String.valueOf(colind);
return alias + ".$" + column;
- } else if (op instanceof NotExpression) {
- String expr = getTestExpression(((NotExpression) op).getExpression());
- return braketize("not " + expr);
} else {
throw new FrontendException("Unsupported conversion of LogicalExpression to Expression: " + op.getName());
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestNewPlanColumnPrune.java Fri Feb 24 03:34:37 2017
@@ -454,34 +454,6 @@ public class TestNewPlanColumnPrune {
}
}
- @Test
- public void testNoAddForeach() throws Exception {
- // PIG-5055
- // Need to make sure that it does not add foreach
- // that drops all the fields from B2.
- String query = "A = load 'd.txt' as (a0:int, a1:int, a2:int);" +
- "B = load 'd.txt' as (b0:int, b1:int, b2:int);" +
- "B2 = FILTER B by b0 == 0;" +
- "C = join A by (1), B2 by (1) ;" +
- "D = FOREACH C GENERATE A::a1, A::a2;" +
- "store D into 'empty';";
-
- LogicalPlan newLogicalPlan = buildPlan(query);
-
- PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
- optimizer.optimize();
- System.err.println(newLogicalPlan);
- Iterator<Operator> iter = newLogicalPlan.getOperators();
- while (iter.hasNext()) {
- Operator o = iter.next();
- LogicalRelationalOperator lro = (LogicalRelationalOperator)o;
- if (lro == null || lro.getAlias() == null) continue;
- if (lro.getAlias().equals("B2")) {
- assertNotNull(lro.getSchema());
- }
- }
- }
-
public class MyPlanOptimizer extends LogicalPlanOptimizer {
protected MyPlanOptimizer(OperatorPlan p, int iterations) {
Modified: pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestPOGenerate.java Fri Feb 24 03:34:37 2017
@@ -21,10 +21,8 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -48,7 +46,6 @@ public class TestPOGenerate {
DataBag cogroup;
DataBag partialFlatten;
DataBag simpleGenerate;
- DataBag mapFlatten;
Random r = new Random();
BagFactory bf = BagFactory.getInstance();
TupleFactory tf = TupleFactory.getInstance();
@@ -57,25 +54,10 @@ public class TestPOGenerate {
public void setUp() throws Exception {
Tuple [] inputA = new Tuple[4];
Tuple [] inputB = new Tuple[4];
- Tuple [] inputC = new Tuple[4];
for(int i = 0; i < 4; i++) {
inputA[i] = tf.newTuple(2);
inputB[i] = tf.newTuple(1);
- inputC[i] = tf.newTuple(2);
}
- Map map0 = new HashMap<String,String>();
- Map map1 = new HashMap<String,String>();
- Map map2 = new HashMap<String,String>();
- Map map3 = new HashMap<String,String>();
- map0.put("A","");
- map0.put("B","");
- map1.put("A","a");
- map1.put("B","b");
- map2.put("A","aa");
- map2.put("B","bb");
- map3.put("A","aaa");
- map3.put("B","bbb");
-
inputA[0].set(0, 'a');
inputA[0].set(1, '1');
inputA[1].set(0, 'b');
@@ -88,15 +70,6 @@ public class TestPOGenerate {
inputB[1].set(0, 'b');
inputB[2].set(0, 'a');
inputB[3].set(0, 'd');
- inputC[0].set(0, 0);
- inputC[0].set(1, map0);
- inputC[1].set(0, 1);
- inputC[1].set(1, map1);
- inputC[2].set(0, 2);
- inputC[2].set(1, map2);
- inputC[3].set(0, 3);
- inputC[3].set(1, map3);
-
DataBag cg11 = bf.newDefaultBag();
cg11.add(inputA[0]);
cg11.add(inputA[2]);
@@ -146,22 +119,15 @@ public class TestPOGenerate {
tPartial[3].append(emptyBag);
partialFlatten = bf.newDefaultBag();
- for (int i = 0; i < 4; ++i) {
+ for(int i = 0; i < 4; ++i) {
partialFlatten.add(tPartial[i]);
}
simpleGenerate = bf.newDefaultBag();
- for (int i = 0; i < 4; ++i) {
+ for(int i = 0; i < 4; ++i) {
simpleGenerate.add(inputA[i]);
}
-
- mapFlatten = bf.newDefaultBag();
- for (int i = 0; i < inputC.length; ++i) {
- mapFlatten.add(inputC[i]);
- }
-
-
//System.out.println("Cogroup : " + cogroup);
//System.out.println("Partial : " + partialFlatten);
//System.out.println("Simple : " + simpleGenerate);
@@ -282,49 +248,4 @@ public class TestPOGenerate {
assertEquals(simpleGenerate.size(), count);
}
-
- @Test
- public void testMapFlattenGenerate() throws Exception {
- ExpressionOperator prj1 = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
- ExpressionOperator prj2 = new POProject(new OperatorKey("", r.nextLong()), -1, 1);
- prj1.setResultType(DataType.INTEGER);
- prj2.setResultType(DataType.MAP);
- List<Boolean> toBeFlattened = new LinkedList<Boolean>();
- toBeFlattened.add(false);
- toBeFlattened.add(true);
- PhysicalPlan plan1 = new PhysicalPlan();
- plan1.add(prj1);
- PhysicalPlan plan2 = new PhysicalPlan();
- plan2.add(prj2);
- List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
- inputs.add(plan1);
- inputs.add(plan2);
- PhysicalOperator poGen = new POForEach(new OperatorKey("", r.nextLong()), 1, inputs, toBeFlattened);
-
- List<String> obtained = new LinkedList<String>();
- for (Tuple t : mapFlatten) {
- poGen.attachInput(t);
- Result output = poGen.getNextTuple();
- while(output.result != null && output.returnStatus != POStatus.STATUS_EOP) {
- //System.out.println(output.result);
- obtained.add(((Tuple) output.result).toString());
- output = poGen.getNextTuple();
- }
- }
-
- int count = 0;
- for (Tuple t : mapFlatten) {
- Tuple expected = tf.newTuple(3);
- expected.set(0, t.get(0));
- for (Object entryObj : ((Map)t.get(1)).entrySet()){
- Map.Entry entry = ((Map.Entry)entryObj);
- expected.set(1, entry.getKey());
- expected.set(2, entry.getValue());
- assertTrue(obtained.contains(expected.toString()));
- ++count;
- }
- }
- assertEquals(mapFlatten.size()*2, count);
-
- }
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestParamSubPreproc.java Fri Feb 24 03:34:37 2017
@@ -19,7 +19,6 @@
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -43,7 +42,6 @@ import org.apache.pig.ExecType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
import org.apache.pig.tools.parameters.ParseException;
-import org.apache.pig.tools.pigstats.PigStats;
import org.junit.Test;
public class TestParamSubPreproc {
@@ -54,9 +52,24 @@ public class TestParamSubPreproc {
private FileInputStream pigExResultStream;
private String basedir = "test/org/apache/pig/test/data";
- private void compareResults(InputStream expected, InputStream result) throws IOException {
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(result));
+ /* Test case 1
+ * Use a parameter within a pig script and provide value on the command line.
+ */
+ @Test
+ public void testCmdlineParam() throws Exception{
+ log.info("Starting test testCmdlineParam() ...");
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"date=20080228"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
String exLine;
String resLine;
@@ -76,26 +89,6 @@ public class TestParamSubPreproc {
inExpected.close();
inResult.close();
- }
-
- /* Test case 1
- * Use a parameter within a pig script and provide value on the command line.
- */
- @Test
- public void testCmdlineParam() throws Exception{
- log.info("Starting test testCmdlineParam() ...");
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"date=20080228"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
-
- compareResults(pigExResultStream, pigResultStream);
log.info("Done");
@@ -117,8 +110,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -140,8 +152,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -205,8 +236,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -230,9 +280,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -253,8 +321,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -277,9 +364,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -301,9 +406,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -323,9 +446,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -346,9 +487,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -368,9 +527,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
}
/* Test case 15,16
@@ -438,8 +615,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
/* Test case 21
@@ -458,8 +654,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
@@ -478,9 +693,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
}
/* Test case 23
@@ -499,8 +732,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
}
/* Test case 24
@@ -519,9 +771,28 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultMulDecs.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
- }
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ }
/* Test case 25
* Use %default to define param values
@@ -538,8 +809,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
/* Test case 26
@@ -557,8 +847,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
}
@@ -577,8 +886,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
}
/* Test case 29
@@ -596,8 +924,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
/* Test case 30
@@ -615,8 +962,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult2.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
/* Test case 31
@@ -635,8 +1001,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -657,9 +1042,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
- compareResults(pigExResultStream, pigResultStream);
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -680,8 +1083,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult3.txt");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -701,8 +1123,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultComment.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
/* Test case
@@ -771,7 +1212,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output26.pig");
InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
- compareResults(expected, pigResultStream);
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
log.info("Done");
}
@@ -791,8 +1252,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDollarSign.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
- compareResults(pigExResultStream, pigResultStream);
+ inExpected.close();
+ inResult.close();
}
@Test
@@ -809,8 +1289,28 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult6.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
- compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -826,9 +1326,27 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- compareResults(pigExResultStream, pigResultStream);
- log.info("Done");
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
}
@Test
@@ -880,351 +1398,6 @@ public class TestParamSubPreproc {
assertEquals(resultContent, "daniel\t10\njenny\t20\n");
}
- @Test
- public void testCommandLineParamOverwritingDefault() throws Exception {
- log.info("Starting test testCommandLineParamOverwritingDefault()");
- File inputFile = Util.createFile(
- "runinput",
- new String[] { "daniel\t10",
- "jenny\t20;"});
- File output1 = File.createTempFile("output1_", "");
- output1.delete();
-
- File script1 = Util.createFile("runscript1.pig",
- new String[] { "%default output /invalidpathThatShouldFail;",
- "a = load 'runinput';",
- "store a into '$output';"});
-
- PigStats stats = org.apache.pig.PigRunner.run(new String[] {
- "-x", Util.getLocalTestMode().toString(),
- "-p", "output=" + output1.getAbsolutePath(),
- script1.getAbsolutePath()} , null);
- Util.deleteDirectory(output1);
-
- assertTrue("job should succeed", stats.isSuccessful());
- assertTrue("Default param should be overridden by the commandline param",
- output1.getAbsolutePath().endsWith(stats.getOutputNames().get(0)));
- }
-
- @Test
- public void testRunWithParamOverwritingDefault() throws Exception {
- log.info("Starting test testScopeOfParamWithRunCommand()");
- File inputFile = Util.createFile(
- "runinput",
- new String[] { "daniel\t10",
- "jenny\t20;"});
- File output1 = File.createTempFile("output1_", "");
- File output2 = File.createTempFile("output2_", "");
- output1.delete();
- output2.delete();
-
-
- File script1 = Util.createFile("runscript1.pig",
- new String[] { "%default output '" + output2.getAbsolutePath() + "';",
- "a = load 'runinput';",
- "store a into '$output';"});
-
- File mainscript = Util.createFile("mainscript.pig",
- new String[] {"run -param output=" + output1.getAbsolutePath()
- + " " + script1.getAbsolutePath() + ";"});
-
-
- PigStats stats = org.apache.pig.PigRunner.run(new String[] {
- "-x", Util.getLocalTestMode().toString(),
- mainscript.getAbsolutePath()} , null);
- Util.deleteDirectory(output1);
- Util.deleteDirectory(output2);
-
- assertTrue("job should succeed", stats.isSuccessful());
- assertEquals("There should only be 1 output.",
- 1, stats.getOutputNames().size());
- assertEquals("Output name should be from output1 and not output2",
- output1.getAbsolutePath(),
- stats.getOutputLocations().get(0));
- }
-
- @Test
- public void testScopeOfParamWithRunCommand() throws Exception {
- log.info("Starting test testScopeOfParamWithRunCommand()");
- File inputFile = Util.createFile(
- "runinput",
- new String[] { "daniel\t10",
- "jenny\t20;"});
- File output1 = File.createTempFile("output1_", "");
- File output2 = File.createTempFile("output2_", "");
- output1.delete();
- output2.delete();
-
- File script1 = Util.createFile("runscript1.pig",
- new String[] { "%default output '" + output1.getAbsolutePath() + "';",
- "a = load 'runinput';",
- "store a into '$output';"});
-
- File script2 = Util.createFile("runscript2.pig",
- new String[] { "%default output '" + output2.getAbsolutePath() + "';",
- "a = load 'runinput';",
- "store a into '$output';"});
-
- File mainscript = Util.createFile("mainscript.pig",
- new String[] { "run " + script1.getAbsolutePath() + ";",
- "run " + script2.getAbsolutePath() + ";" });
-
- PigStats stats = org.apache.pig.PigRunner.run(new String[] {
- "-x", Util.getLocalTestMode().toString(),
- mainscript.getAbsolutePath()} , null);
- Util.deleteDirectory(output1);
- Util.deleteDirectory(output2);
-
- assertTrue("job should succeed", stats.isSuccessful());
- assertNotEquals("Two output paths should differ",
- stats.getOutputNames().get(0), stats.getOutputNames().get(1));
- assertEquals("Each output should contain 2 records",
- 2, stats.getOutputStats().get(0).getNumberRecords());
- assertEquals("Each output should contain 2 records",
- 2, stats.getOutputStats().get(1).getNumberRecords());
- }
-
- @Test
- public void testScopeOfParamWithNestedRunCommand() throws Exception {
- log.info("Starting test testScopeOfParamWithRunCommand()");
- File inputFile = Util.createFile(
- "runinput",
- new String[] { "daniel\t10",
- "jenny\t20;"});
- /*
- * script1 sets a=1, b=2, c=3; calls script2
- * script2 sets b=22 (by -param); calls script3
- * script3 sets c=333; saves $a$b$c (122333)
- * script2 saves $a$b$c (1223)
- * script1 saves $a$b$c (123)
- */
- File script3 = Util.createFile("runscript3.pig",
- new String[] { "%declare c '333';",
- "a = load 'runinput';",
- "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"});
-
- File script2 = Util.createFile("runscript2.pig",
- new String[] { "run " + script3.getAbsolutePath() + ";",
- "a = load 'runinput';",
- "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"});
-
- File script1 = Util.createFile("runscript1.pig",
- new String[] { "%declare a '1';",
- "%declare b '2';",
- "%declare c '3';",
- "run -param b=22 " + script2.getAbsolutePath() + ";",
- "a = load 'runinput';",
- "store a into 'testScopeOfParamWithNestedRunCommand${a}${b}${c}';"});
-
- PigStats stats = org.apache.pig.PigRunner.run(new String[] {
- "-x", Util.getLocalTestMode().toString(),
- script1.getAbsolutePath()} , null);
-
- for( String output : stats.getOutputNames() ) {
- assertTrue(output.contains("testScopeOfParamWithNestedRunCommand"));
- Util.deleteDirectory(new File(output));
- }
- assertTrue("job should succeed", stats.isSuccessful());
- assertEquals("There should be three outputs.", 3, stats.getOutputNames().size());
-
- for( String expectedoutput : new String [] {"testScopeOfParamWithNestedRunCommand123",
- "testScopeOfParamWithNestedRunCommand1223",
- "testScopeOfParamWithNestedRunCommand122333"} ) {
- boolean found=false;
- for( String output : stats.getOutputNames() ) {
- if( output.endsWith(expectedoutput) ) {
- found=true;
- }
- }
- assertTrue("Output " + expectedoutput + " should exist.", found);
- }
- }
-
- /* This currently does not work since PigMacro only picks the
- * param setting from the root script (script1)
- * To revisit after Grunt moves to ANTLR in PIG-2597.
- * Tracking in PIG-5028.
- *
-
- @Test
- public void testScopeOfParamWithMacro() throws Exception {
- log.info("Starting test testScopeOfParamWithMacro()");
- File inputFile = Util.createFile(
- "runinput",
- new String[] { "daniel\t10",
- "jenny\t20;"});
- File macro = Util.createFile("testmacro.pig",
- new String[] { "DEFINE mymacro (A) RETURNS void {",
- "store $A into 'testScopeOfParamWithMacro${a}${b}${c}';",
- "};"});
-
- File script3 = Util.createFile("runscript3.pig",
- new String[] { "%declare c '333';"});
-
- File script2 = Util.createFile("runscript2.pig",
- new String[] { "%declare b '22';",
- "import '" + macro.getAbsolutePath() + "';",
- "a = load 'runinput';",
- "mymacro(a);",
- "exec " + script3.getAbsolutePath() + ";"});
-
- File script1 = Util.createFile("runscript1.pig",
- new String[] { "%declare a '1';",
- "%declare b '2';",
- "%declare c '3';",
- "exec " + script2.getAbsolutePath() + ";"});
-
- PigStats stats = org.apache.pig.PigRunner.run(new String[] {
- "-x", Util.getLocalTestMode().toString(),
- script1.getAbsolutePath()} , null);
-
- assertTrue("job should succeed", stats.isSuccessful());
- Util.deleteDirectory(new File(stats.getOutputNames().get(0)));
- assertEquals("There should be only 1 output.", 1, stats.getOutputNames().size());
- assertTrue("Expected output testScopeOfParamWithMacro1223 but got " + stats.getOutputNames().get(0),
- stats.getOutputNames().get(0).equals("testScopeOfParamWithMacro1223"));
- }
- */
-
-
- /*
- * Test parameter substition when register contains /* globbing
- */
- @Test
- public void testSubstitutionWithRegisterGlobbing() throws Exception{
- log.info("Starting test testSubstitutionWithRegisterGlobbing()");
- final String queryString =
- "register /abc/$regdir/*.jar;\n" +
- "A = LOAD '$input' USING PigStorage ();\n" +
- "STORE A INTO '$output';\n" +
- " /* comment that would make register globbing to be part of the multi-line comment */\n";
-
-
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(
- new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"input = 'input.txt'", "output = 'output.txt'", "regdir = 'def'"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
-
- String expectedString = queryString.replaceAll("\\$input","input.txt")
- .replaceAll("\\$output","output.txt")
- .replaceAll("\\$regdir","def");
- InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
-
- compareResults(expected, pigResultStream);
-
- log.info("Done");
- }
-
- /*
- * Test parameter substition when load contains /* globbing
- */
- @Test
- public void testSubstitutionWithLoadGlobbing() throws Exception{
- log.info("Starting test testSubstitutionWithLoadGlobbing()");
- final String queryString =
- "A = LOAD '/zzz/*' USING PigStorage ();\n" +
- "STORE A INTO '$output';\n" +
- " /* comment that would make register globbing to be part of the multi-line comment */\n";
-
-
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(
- new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"output = 'output.txt'"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
-
- String expectedString = queryString.replaceAll("\\$output","output.txt");
- InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
-
- compareResults(expected, pigResultStream);
-
- log.info("Done");
- }
-
- @Test
- public void testSubstitutionWithRedeclaration() throws Exception{
- log.info("Starting test testSubstitutionWithRedeclaration()");
- final String queryString =
- "%declare output '/tmp/abc';\n" +
- "%declare actualoutput '$output.out';\n" +
- "A = load 'input.txt' ;\n" +
- "store A into '$actualoutput';\n" +
- "%declare output '/tmp/def';\n" +
- "%declare actualoutput '$output.out';\n" +
- "store A into '$actualoutput';";
-
-
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(
- new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"output = 'output.txt'"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
-
- String expectedString = queryString.replaceAll("%declare [0-9a-zA-Z.'/\\$; ]*\n",";\n")
- .replaceAll("\\$","")
- .replaceFirst("actualoutput","/tmp/abc.out")
- .replaceFirst("actualoutput","/tmp/def.out");
- InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
-
- compareResults(expected, pigResultStream);
-
- log.info("Done");
- }
-
- @Test
- public void testSubstitutionWithRedeclaredShell() throws Exception{
- log.info("Starting test testSubstitutionWithRedeclaredShell()");
- final String queryString =
- "A = load 'input.txt' ;\n" +
- "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" +
- "store A into '$now';\n" +
- "%declare now `bash -c \"date +'%Y%m%d_%H:%M:%S'; sleep 1;\"`;\n" +
- "store A into '$now';\n";
-
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(
- new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"output = 'output.txt'"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- BufferedReader pigresult = new BufferedReader(new InputStreamReader(new FileInputStream(basedir + "/output1.pig")));
-
-
- String [] filenames = new String [2];
- int index=0;
- String line;
- while ((line = pigresult.readLine())!=null) {
- if( line.startsWith("store A into") ) {
- filenames[index++] = line.split(" ")[3];
- }
- }
-
- assertEquals("There should be 2 store statements", 2, index);
- assertNotEquals("Identical shell param should be reexecuted.",
- filenames[0],
- filenames[1]);
- log.info("Done");
- }
-
@SuppressWarnings("resource")
private BufferedReader WithConditionalReplacement(String filename, String orig, String dest, boolean replace) throws IOException {
BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename));
Modified: pig/branches/spark/test/org/apache/pig/test/TestPigContext.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestPigContext.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestPigContext.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestPigContext.java Fri Feb 24 03:34:37 2017
@@ -27,7 +27,6 @@ import java.util.List;
import java.util.Properties;
import java.util.Random;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
@@ -271,7 +270,7 @@ public class TestPigContext {
assertEquals(JOB_TRACKER,
pigServer.getPigContext().getProperties().getProperty(MRConfiguration.JOB_TRACKER));
assertEquals(FS_NAME,
- pigServer.getPigContext().getProperties().getProperty(FileSystem.FS_DEFAULT_NAME_KEY));
+ pigServer.getPigContext().getProperties().getProperty("fs.default.name"));
assertEquals(TMP_DIR_PROP,
pigServer.getPigContext().getProperties().getProperty("hadoop.tmp.dir"));
}