You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/05/14 20:01:33 UTC
svn commit: r1482501 - in /pig/trunk: ./ test/org/apache/pig/test/
Author: daijy
Date: Tue May 14 18:01:33 2013
New Revision: 1482501
URL: http://svn.apache.org/r1482501
Log:
PIG-3026: Pig checked-in baseline comparisons need a pre-filter to address OS-specific newline differences
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java
pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
pig/trunk/test/org/apache/pig/test/TestOptimizeLimit.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 14 18:01:33 2013
@@ -178,6 +178,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3026: Pig checked-in baseline comparisons need a pre-filter to address OS-specific newline differences (dwann via daijy)
+
PIG-3025: TestPruneColumn unit test - SimpleEchoStreamingCommand perl inline script needs simplification (dwann via daijy)
PIG-2955: Fix bunch of Pig e2e tests on Windows (daijy)
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java Tue May 14 18:01:33 2013
@@ -1058,13 +1058,15 @@ public class TestEvalPipelineLocal {
// Filter out the random number generated on hash
realPlan = realPlan.replaceAll("\\d{3,}", "");
+ String goldenPlanClean = Util.standardizeNewline(goldenPlan);
+ String realPlanClean = Util.standardizeNewline(realPlan);
System.out.println("-----------golden");
- System.out.println(goldenPlan);
+ System.out.println(goldenPlanClean);
System.out.println("-----------");
- System.out.println(realPlan);
+ System.out.println(realPlanClean);
- Assert.assertEquals(realPlan, goldenPlan);
+ Assert.assertEquals(realPlanClean, goldenPlanClean);
}
public static class SetLocationTestLoadFunc extends PigStorage {
@@ -1087,7 +1089,7 @@ public class TestEvalPipelineLocal {
@Test
public void testSetLocationCalledInFE() throws Exception {
File f1 = createFile(new String[]{"a","b"});
- pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ pigServer.registerQuery("a = load '" + Util.generateURI(Util.encodeEscape(f1.toString()), pigServer.getPigContext())
+ "' using " + SetLocationTestLoadFunc.class.getName()
+ "();");
pigServer.registerQuery("b = order a by $0;");
@@ -1100,7 +1102,7 @@ public class TestEvalPipelineLocal {
@Test
public void testGroupByTuple() throws Exception {
File f1 = createFile(new String[]{"1\t2\t3","4\t5\t6"});
- pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ pigServer.registerQuery("a = load '" + Util.generateURI(Util.encodeEscape(f1.toString()), pigServer.getPigContext())
+ "' as (x:int, y:int, z:int);");
pigServer.registerQuery("b = foreach a generate TOTUPLE(x, y) as t, z;");
pigServer.registerQuery("c = group b by t;");
@@ -1114,7 +1116,7 @@ public class TestEvalPipelineLocal {
// See PIG-3060
public void testFlattenEmptyBag() throws Exception {
File f1 = createFile(new String[]{"2\t{}","3\t{(1),(2)}", "4\t{}"});
- pigServer.registerQuery("A = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ pigServer.registerQuery("A = load '" + Util.generateURI(Util.encodeEscape(f1.toString()), pigServer.getPigContext())
+ "' as (a0:int, a1:bag{(t:chararray)});");
pigServer.registerQuery("B = group A by a0;");
pigServer.registerQuery("C = foreach B { c1 = foreach A generate FLATTEN(a1); generate COUNT(c1);};");
Modified: pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java Tue May 14 18:01:33 2013
@@ -89,7 +89,6 @@ public class TestLogToPhyCompiler {
PigServer pigServer = null;
private static final int MAX_SIZE = 100000;;
-
@Before
public void setUp() throws Exception {
@@ -112,6 +111,7 @@ public class TestLogToPhyCompiler {
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = Util.standardizeNewline(compiledPlan);
compiledPlan = removedUnaffectingChanges(compiledPlan);
generateGolden(goldenFile, compiledPlan);
@@ -134,6 +134,7 @@ public class TestLogToPhyCompiler {
int len = fis.read(b);
fis.close();
String goldenPlan = new String(b, 0, len);
+ goldenPlan = Util.standardizeNewline(goldenPlan);
goldenPlan = removedUnaffectingChanges(goldenPlan);
return goldenPlan;
}
@@ -155,7 +156,8 @@ public class TestLogToPhyCompiler {
System.out.println();
System.out.println(compiledPlan);
System.out.println("-------------" + testName);
- assertEquals(compiledPlan, goldenPlan);
+
+ assertEquals(goldenPlan, compiledPlan);
}
@Test // Commented out due to PIG-2020
@@ -266,7 +268,9 @@ public class TestLogToPhyCompiler {
//System.out.println("Length of first plan = " + len + " of second = " + test);
String goldenPlan1 = new String(b1, 0, len);
String goldenPlan2 = new String(b2, 0, len);
+ goldenPlan1 = Util.standardizeNewline(goldenPlan1);
goldenPlan1 = removedUnaffectingChanges(goldenPlan1);
+ goldenPlan2 = Util.standardizeNewline(goldenPlan2);
goldenPlan2 = removedUnaffectingChanges(goldenPlan2);
System.out.println();
@@ -320,7 +324,9 @@ public class TestLogToPhyCompiler {
//System.out.println("Length of first plan = " + len + " of second = " + test + " Length of compiled plan = " + compiledPlan.length());
String goldenPlan1 = new String(b1, 0, len);
String goldenPlan2 = new String(b2, 0, len);
+ goldenPlan1 = Util.standardizeNewline(goldenPlan1);
goldenPlan1 = removedUnaffectingChanges(goldenPlan1);
+ goldenPlan2 = Util.standardizeNewline(goldenPlan2);
goldenPlan2 = removedUnaffectingChanges(goldenPlan2);
System.out.println();
Modified: pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Tue May 14 18:01:33 2013
@@ -1125,7 +1125,10 @@ public class TestMRCompiler {
System.out.println("Golden");
System.out.println("<<<" + goldenPlan + ">>>");
System.out.println("-------------");
- assertEquals(TestHelper.sortUDFs(removeSignature(goldenPlan)), TestHelper.sortUDFs(removeSignature(compiledPlan)));
+
+ String goldenPlanClean = Util.standardizeNewline(goldenPlan);
+ String compiledPlanClean = Util.standardizeNewline(compiledPlan);
+ assertEquals(TestHelper.sortUDFs(removeSignature(goldenPlanClean)), TestHelper.sortUDFs(removeSignature(compiledPlanClean)));
}
/**
Modified: pig/trunk/test/org/apache/pig/test/TestOptimizeLimit.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestOptimizeLimit.java?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestOptimizeLimit.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestOptimizeLimit.java Tue May 14 18:01:33 2013
@@ -63,17 +63,19 @@ public class TestOptimizeLimit {
void compareWithGoldenFile(LogicalPlan plan, String filename) throws Exception {
String actualPlan = printLimitGraph(plan);
+ String actualPlanClean = Util.standardizeNewline(actualPlan + "\n");
System.out.println("We get:");
- System.out.println(actualPlan);
+ System.out.println(actualPlanClean);
FileInputStream fis = new FileInputStream(filename);
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
+ String goldenPlanClean = Util.standardizeNewline(goldenPlan);
System.out.println("Expected:");
- System.out.println(goldenPlan);
+ System.out.println(goldenPlanClean);
- Assert.assertEquals(goldenPlan, actualPlan + "\n");
+ Assert.assertEquals(goldenPlanClean, actualPlanClean);
}
public static String printLimitGraph(LogicalPlan plan) throws IOException {
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1482501&r1=1482500&r2=1482501&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Tue May 14 18:01:33 2013
@@ -277,6 +277,19 @@ public class Util {
}
/**
+ * Helper to convert \r\n to \n for cross-platform string
+ * matching with checked-in baselines.
+ *
+ * @param origPath original string
+ * @return String newline-standardized string
+ * @throws IOException
+ */
+ static public String standardizeNewline(String origPath)
+ {
+ return origPath.replaceAll("\r\n", "\n");
+ }
+
+ /**
* Helper to create a temporary file with given input data for use in test cases.
*
* @param tmpFilenamePrefix file-name prefix