You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2010/09/14 19:46:31 UTC
svn commit: r997012 - in /hadoop/pig/branches/branch-0.8: CHANGES.txt
test/org/apache/pig/test/TestNativeMapReduce.java
Author: thejas
Date: Tue Sep 14 17:46:31 2010
New Revision: 997012
URL: http://svn.apache.org/viewvc?rev=997012&view=rev
Log:
PIG-1589: add test cases for mapreduce operator which use distributed cache (thejas)
Modified:
hadoop/pig/branches/branch-0.8/CHANGES.txt
hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestNativeMapReduce.java
Modified: hadoop/pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/CHANGES.txt?rev=997012&r1=997011&r2=997012&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.8/CHANGES.txt Tue Sep 14 17:46:31 2010
@@ -26,6 +26,8 @@ PIG-1249: Safe-guards against misconfigu
IMPROVEMENTS
+PIG-1589: add test cases for mapreduce operator which use distributed cache (thejas)
+
PIG-1575: Complete the migration of optimization rule PushUpFilter including
missing test cases (xuefuz via daijy)
Modified: hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestNativeMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestNativeMapReduce.java?rev=997012&r1=997011&r2=997012&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestNativeMapReduce.java (original)
+++ hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestNativeMapReduce.java Tue Sep 14 17:46:31 2010
@@ -21,18 +21,17 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Random;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.tools.pigstats.PigStats;
import org.junit.AfterClass;
import org.junit.Before;
@@ -53,7 +52,12 @@ public class TestNativeMapReduce {
// http://svn.apache.org/repos/asf/hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/WordCount.java:816822
private String jarFileName = "test//org/apache/pig/test/data/TestWordCount.jar";
private String exp_msg_prefix = "Check if expected results contains: ";
- final static String INPUT_FILE = "TestMapReduceInputFile";
+ final static String INPUT_FILE = "TestNMapReduceInputFile";
+ /**
+ *stop word file - used to test distributed cache usage, words in this
+ * file if specified will be skipped by the wordcount udf
+ */
+ final static String STOPWORD_FILE = "TestNMapReduceStopwFile";
static MiniCluster cluster = MiniCluster.buildCluster();
private PigServer pigServer = null;
@@ -72,19 +76,38 @@ public class TestNativeMapReduce {
"two",
"three"
};
+ //for stop word file
+ String[] stopw = {
+ "one"
+ };
+
Util.createInputFile(cluster, INPUT_FILE, input);
+ Util.createLocalInputFile(STOPWORD_FILE, stopw);
}
+ // createWordCountJar(){
+ // // its a manual process
+ // javac -cp build/ivy/lib/Pig/hadoop-core-0.20.2.jar:build/ivy/lib/Pig/commons-cli-1.2.jar test/org/apache/pig/test/utils/WordCount.java
+ // cd test/
+ // jar -cf WordCount.jar org/apache/pig/test/utils/WordCount*class
+ // mv WordCount.jar org/apache/pig/test/data/TestWordCount.jar
+ //
+ //
+ //}
+
@Before
public void setUp() throws Exception{
pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
//createWordCountJar();
}
+
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
Util.deleteFile(cluster, INPUT_FILE);
+ new File(STOPWORD_FILE).delete();
cluster.shutDown();
}
@@ -94,16 +117,20 @@ public class TestNativeMapReduce {
public void testNativeMRJobSimple() throws Exception{
try{
Collection<String> results = new HashSet<String>();
- results.add("(one,1)");
results.add("(two,2)");
results.add("(three,3)");
pigServer.setBatchOn();
pigServer.registerQuery("A = load '" + INPUT_FILE + "';");
+
+
+ //also test distributed cache using the stopwords file for udf
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobSimple_input' "+
"Load 'table_testNativeMRJobSimple_output' "+
- "`WordCount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;");
+ "`org.apache.pig.test.utils.WordCount -files " + STOPWORD_FILE +
+ " table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output " +
+ STOPWORD_FILE + "`;");
pigServer.registerQuery("Store B into 'table_testNativeMRJobSimpleDir';");
List<ExecJob> execJobs = pigServer.executeBatch();
@@ -131,10 +158,6 @@ public class TestNativeMapReduce {
t = iter.next();
assertTrue(exp_msg_prefix + t, results.contains(t.toString()));
- assertTrue("iter.hasNext()",iter.hasNext());
- t = iter.next();
- assertTrue(exp_msg_prefix + t, results.contains(t.toString()));
-
assertFalse(iter.hasNext());
// We have to manually delete intermediate mapreduce files
@@ -152,10 +175,6 @@ public class TestNativeMapReduce {
t = iter.next();
assertTrue(exp_msg_prefix + t, results.contains(t.toString()));
- assertTrue("iter.hasNext()",iter.hasNext());
- t = iter.next();
- assertTrue(exp_msg_prefix + t, results.contains(t.toString()));
-
assertFalse(iter.hasNext());
}
finally{
@@ -185,7 +204,7 @@ public class TestNativeMapReduce {
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobSimple_input' "+
"Load 'table_testNativeMRJobSimple_output' "+
- "`WordCount table_testNativeMRJobSimple_input " + INPUT_FILE + "`;");
+ "`org.apache.pig.test.utils.WordCount table_testNativeMRJobSimple_input " + INPUT_FILE + "`;");
pigServer.registerQuery("Store B into 'table_testNativeMRJobSimpleDir';");
// List<ExecJob> execJobs = pigServer.executeBatch();
@@ -217,7 +236,7 @@ public class TestNativeMapReduce {
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobMultiStoreOnPred_input' "+
"Load 'table_testNativeMRJobMultiStoreOnPred_output' "+
- "`WordCount table_testNativeMRJobMultiStoreOnPred_input table_testNativeMRJobMultiStoreOnPred_output`;");
+ "`org.apache.pig.test.utils.WordCount table_testNativeMRJobMultiStoreOnPred_input table_testNativeMRJobMultiStoreOnPred_output`;");
pigServer.registerQuery("Store B into 'table_testNativeMRJobMultiStoreOnPredDir';");
pigServer.executeBatch();
@@ -282,11 +301,11 @@ public class TestNativeMapReduce {
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobMultiQueryOpt_inputB' "+
"Load 'table_testNativeMRJobMultiQueryOpt_outputB' "+
- "`WordCount table_testNativeMRJobMultiQueryOpt_inputB table_testNativeMRJobMultiQueryOpt_outputB`;");
+ "`org.apache.pig.test.utils.WordCount table_testNativeMRJobMultiQueryOpt_inputB table_testNativeMRJobMultiQueryOpt_outputB`;");
pigServer.registerQuery("C = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobMultiQueryOpt_inputC' "+
"Load 'table_testNativeMRJobMultiQueryOpt_outputC' "+
- "`WordCount table_testNativeMRJobMultiQueryOpt_inputC table_testNativeMRJobMultiQueryOpt_outputC`;");
+ "`org.apache.pig.test.utils.WordCount table_testNativeMRJobMultiQueryOpt_inputC table_testNativeMRJobMultiQueryOpt_outputC`;");
Iterator<Tuple> iter = pigServer.openIterator("C");
Tuple t;
@@ -341,7 +360,7 @@ public class TestNativeMapReduce {
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobTypeCastInserter_input' "+
"Load 'table_testNativeMRJobTypeCastInserter_output' as (name:chararray, count: int)"+
- "`WordCount table_testNativeMRJobTypeCastInserter_input table_testNativeMRJobTypeCastInserter_output`;");
+ "`org.apache.pig.test.utils.WordCount table_testNativeMRJobTypeCastInserter_input table_testNativeMRJobTypeCastInserter_output`;");
pigServer.registerQuery("C = foreach B generate count+1;");
Iterator<Tuple> iter = pigServer.openIterator("C");