You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2012/11/08 05:47:06 UTC
svn commit: r1406922 - in /pig/branches/branch-0.11: CHANGES.txt
test/org/apache/pig/test/TestDataModel.java
test/org/apache/pig/test/TestMRCompiler.java
test/org/apache/pig/test/TestPruneColumn.java
test/org/apache/pig/test/utils/TestHelper.java
Author: cheolsoo
Date: Thu Nov 8 04:47:05 2012
New Revision: 1406922
URL: http://svn.apache.org/viewvc?rev=1406922&view=rev
Log:
PIG-2405: some unit test case failed with open JDK (fang fang chen via cheolsoo)
Modified:
pig/branches/branch-0.11/CHANGES.txt
pig/branches/branch-0.11/test/org/apache/pig/test/TestDataModel.java
pig/branches/branch-0.11/test/org/apache/pig/test/TestMRCompiler.java
pig/branches/branch-0.11/test/org/apache/pig/test/TestPruneColumn.java
pig/branches/branch-0.11/test/org/apache/pig/test/utils/TestHelper.java
Modified: pig/branches/branch-0.11/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/CHANGES.txt?rev=1406922&r1=1406921&r2=1406922&view=diff
==============================================================================
--- pig/branches/branch-0.11/CHANGES.txt (original)
+++ pig/branches/branch-0.11/CHANGES.txt Thu Nov 8 04:47:05 2012
@@ -316,6 +316,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2405: some unit test case failed with open JDK (fang fang chen via cheolsoo)
+
PIG-3035: With latest version of hadoop23 pig does not return the correct exception stack trace from backend (rohini)
PIG-3018: Refactor TestScriptLanguage to remove duplication and write script in different files (julien)
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/TestDataModel.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/TestDataModel.java?rev=1406922&r1=1406921&r2=1406922&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/TestDataModel.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/TestDataModel.java Thu Nov 8 04:47:05 2012
@@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.LinkedHashMap;
import org.junit.Test;
@@ -594,7 +595,7 @@ public class TestDataModel extends junit
bag.add(tf.newTuple(new Integer(4)));
bag.add(tf.newTuple(new String("mary had a little lamb")));
- Map<String, Object> map = new HashMap<String, Object>(2);
+ Map<String, Object> map = new LinkedHashMap<String, Object>(2);
map.put(new String("hello"), new String("world"));
map.put(new String("goodbye"), new String("all"));
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/TestMRCompiler.java?rev=1406922&r1=1406921&r2=1406922&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/TestMRCompiler.java Thu Nov 8 04:47:05 2012
@@ -75,6 +75,7 @@ import org.apache.pig.impl.util.Utils;
import org.apache.pig.test.junit.OrderedJUnit4Runner;
import org.apache.pig.test.junit.OrderedJUnit4Runner.TestOrder;
import org.apache.pig.test.utils.GenPhyOp;
+import org.apache.pig.test.utils.TestHelper;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -1121,7 +1122,7 @@ public class TestMRCompiler {
System.out.println("Golden");
System.out.println("<<<" + goldenPlan + ">>>");
System.out.println("-------------");
- assertEquals(removeSignature(goldenPlan), removeSignature(compiledPlan));
+ assertEquals(TestHelper.sortUDFs(removeSignature(goldenPlan)), TestHelper.sortUDFs(removeSignature(compiledPlan)));
}
/**
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/TestPruneColumn.java?rev=1406922&r1=1406921&r2=1406922&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/TestPruneColumn.java Thu Nov 8 04:47:05 2012
@@ -54,6 +54,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
+import org.apache.pig.test.utils.TestHelper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -207,6 +208,9 @@ public class TestPruneColumn extends Tes
{
logMessages.add(line);
}
+ if (logMessages.size() > 0) {
+ logMessages = TestHelper.sortSubFields(logMessages);
+ }
// Check if all messages appear in the log
for (int i=0;i<messages.length;i++)
@@ -1197,7 +1201,7 @@ public class TestPruneColumn extends Tes
assertFalse(iter.hasNext());
assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0",
- "Map key required for A: $1->[key2, key1]"}));
+ "Map key required for A: $1->[key1, key2]"}));
}
@Test
@@ -1284,7 +1288,7 @@ public class TestPruneColumn extends Tes
assertFalse(iter.hasNext());
- assertTrue(checkLogFileMessage(new String[]{"Map key required for A: $0->[key2, key1]"}));
+ assertTrue(checkLogFileMessage(new String[]{"Map key required for A: $0->[key1, key2]"}));
}
@SuppressWarnings("rawtypes")
@@ -1610,7 +1614,7 @@ public class TestPruneColumn extends Tes
assertTrue(iter.hasNext());
Tuple t = iter.next();
- assertTrue(t.toString().equals("([2#1,1#1])"));
+ assertEquals("([1#1, 2#1])", TestHelper.sortString("\\[(.*)\\]", t.toString(), ","));
assertFalse(iter.hasNext());
@@ -2102,8 +2106,8 @@ public class TestPruneColumn extends Tes
Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("event_serve_join")));
- assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[key4, key3]",
- "Map key required for cm_data_raw: $0->[key4, key3, key5]"}));
+ assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[key3, key4]",
+ "Map key required for cm_data_raw: $0->[key3, key4, key5]"}));
}
// See PIG-2535
@@ -2126,8 +2130,8 @@ public class TestPruneColumn extends Tes
pigServer.explain("event_serve_join", System.out);
- assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[event_guid, receive_time, filter_key]",
- "Map key required for raw: $0->[source, p_url, cm_serve_timestamp_ms, cm_serve_id, type]"}));
+ assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[event_guid, filter_key, receive_time]",
+ "Map key required for raw: $0->[cm_serve_id, cm_serve_timestamp_ms, p_url, source, type]"}));
}
}
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/utils/TestHelper.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/utils/TestHelper.java?rev=1406922&r1=1406921&r2=1406922&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/utils/TestHelper.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/utils/TestHelper.java Thu Nov 8 04:47:05 2012
@@ -24,6 +24,11 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Map.Entry;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
@@ -398,4 +403,54 @@ public class TestHelper {
}
+ /**
+ * Find out the string which matches "regex" from "target", and sort the string with spacial
+ * order. The string elements are split by "split".
+ */
+ public static String sortString(String regex, String target, String split) {
+ Pattern p = Pattern.compile(regex);
+ Matcher matcher = p.matcher(target);
+ String original = null;
+ String replaceString = new String();
+
+ if (matcher.find()) {
+ original = matcher.group(1);
+ String[] out = original.split(split);
+ Collections.sort(Arrays.asList(out));
+ for (int j = 0; j < out.length; j++) {
+ replaceString += (j > 0 ? ", " + out[j] : out[j]);
+ }
+ return target.replace(original, replaceString);
+ }
+ return target;
+ }
+
+ /**
+ * Sort UDFs for golden plan
+ */
+ public static String sortUDFs(String goldenString) {
+ String regex = "MapReduce\\([0-9]*\\,(.*)\\) - -[0-9]*\\:";
+ String[] goldenArray = goldenString.split("\n");
+
+ for (int i = 0; i < goldenArray.length; i++) {
+ goldenString = goldenString.replace(goldenArray[i],
+ sortString(regex, goldenArray[i], ","));
+ }
+
+ return goldenString;
+ }
+
+ /**
+ * sort subFields for LogMessages
+ */
+ public static List<String> sortSubFields(List<String> logMessages) {
+ String regex = "\\[(.*)\\]";
+
+ for (int i = 0; i < logMessages.size(); i++) {
+ logMessages.set(i, sortString(regex, logMessages.get(i), ", "));
+ }
+
+ return logMessages;
+ }
+
}