You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/04/11 00:14:18 UTC
svn commit: r646988 [2/3] - in /incubator/pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/executionengine/
src/org/apache/pig/backend/hadoop/datastorage/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/...
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Thu Apr 10 15:14:04 2008
@@ -15,68 +15,71 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.StringTokenizer;
-
-import org.junit.Test;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.PigServer;
-import org.apache.pig.builtin.BinStorage;
-import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.builtin.TextLoader;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.io.PigFile;
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.StringTokenizer;
+
+import org.junit.Test;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.PigServer;
+import org.apache.pig.PigServer.ExecType;
+import org.apache.pig.builtin.BinStorage;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.builtin.TextLoader;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.io.PigFile;
import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
import org.apache.pig.backend.executionengine.ExecException;
-
-import junit.framework.TestCase;
-
-public class TestEvalPipeline extends TestCase {
-
+
+import junit.framework.TestCase;
+
+public class TestEvalPipeline extends TestCase {
+
MiniCluster cluster = MiniCluster.buildCluster();
-
-
- static public class MyBagFunction extends EvalFunc<DataBag>{
- @Override
- public void exec(Tuple input, DataBag output) throws IOException {
- output.add(new Tuple("a"));
- output.add(new Tuple("a"));
- output.add(new Tuple("a"));
-
- }
- }
-
- private File createFile(String[] data) throws Exception{
- File f = File.createTempFile("tmp", "");
- PrintWriter pw = new PrintWriter(f);
- for (int i=0; i<data.length; i++){
- pw.println(data[i]);
- }
- pw.close();
- return f;
+ private PigServer pigServer;
+
+ @Override
+ protected void setUp() throws Exception {
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ }
+
+ static public class MyBagFunction extends EvalFunc<DataBag>{
+ @Override
+ public void exec(Tuple input, DataBag output) throws IOException {
+ output.add(new Tuple("a"));
+ output.add(new Tuple("a"));
+ output.add(new Tuple("a"));
+
+ }
+ }
+
+ private File createFile(String[] data) throws Exception{
+ File f = File.createTempFile("tmp", "");
+ PrintWriter pw = new PrintWriter(f);
+ for (int i=0; i<data.length; i++){
+ pw.println(data[i]);
+ }
+ pw.close();
+ return f;
}
@Test
public void testFunctionInsideFunction() throws Throwable {
- PigServer pigServer = new PigServer(MAPREDUCE);
-
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
-
pigServer.registerQuery("a = load 'file:" + f1 + "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate '1'-'1'/'1';");
Iterator<Tuple> iter = pigServer.openIterator("b");
@@ -85,224 +88,218 @@
assertEquals(iter.next().getAtomField(0).numval(), 0.0);
}
- }
-
- @Test
- public void testJoin() throws Throwable {
- PigServer pigServer = new PigServer(MAPREDUCE);
-
- File f1 = createFile(new String[]{"a:1","b:1","a:1"});
- File f2 = createFile(new String[]{"b","b","a"});
-
- pigServer.registerQuery("a = load 'file:" + f1 + "' using " + PigStorage.class.getName() + "(':');");
- pigServer.registerQuery("b = load 'file:" + f2 + "';");
- pigServer.registerQuery("c = cogroup a by $0, b by $0;");
- pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
-
- Iterator<Tuple> iter = pigServer.openIterator("d");
- int count = 0;
- while(iter.hasNext()){
- Tuple t = iter.next();
- assertTrue(t.getAtomField(0).strval().equals(t.getAtomField(2).strval()));
- count++;
- }
- assertEquals(count, 4);
- }
-
- @Test
- public void testDriverMethod() throws Throwable {
- PigServer pigServer = new PigServer(MAPREDUCE);
- File f = File.createTempFile("tmp", "");
- PrintWriter pw = new PrintWriter(f);
- pw.println("a");
- pw.println("a");
- pw.close();
- pigServer.registerQuery("a = foreach (load 'file:" + f + "') generate '1', flatten(" + MyBagFunction.class.getName() + "(*));");
- pigServer.registerQuery("b = foreach a generate $0, flatten($1);");
- Iterator<Tuple> iter = pigServer.openIterator("a");
- int count = 0;
- while(iter.hasNext()){
- Tuple t = iter.next();
- assertTrue(t.getAtomField(0).strval().equals("1"));
- assertTrue(t.getAtomField(1).strval().equals("a"));
- count++;
- }
- assertEquals(count, 6);
- f.delete();
- }
-
-
- @Test
- public void testMapLookup() throws Throwable {
- PigServer pigServer = new PigServer(MAPREDUCE);
+ }
+
+ @Test
+ public void testJoin() throws Throwable {
+ File f1 = createFile(new String[]{"a:1","b:1","a:1"});
+ File f2 = createFile(new String[]{"b","b","a"});
+
+ pigServer.registerQuery("a = load 'file:" + f1 + "' using " + PigStorage.class.getName() + "(':');");
+ pigServer.registerQuery("b = load 'file:" + f2 + "';");
+ pigServer.registerQuery("c = cogroup a by $0, b by $0;");
+ pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
+
+ Iterator<Tuple> iter = pigServer.openIterator("d");
+ int count = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ assertTrue(t.getAtomField(0).strval().equals(t.getAtomField(2).strval()));
+ count++;
+ }
+ assertEquals(count, 4);
+ }
+
+ @Test
+ public void testDriverMethod() throws Throwable {
+ File f = File.createTempFile("tmp", "");
+ PrintWriter pw = new PrintWriter(f);
+ pw.println("a");
+ pw.println("a");
+ pw.close();
+ pigServer.registerQuery("a = foreach (load 'file:" + f + "') generate '1', flatten(" + MyBagFunction.class.getName() + "(*));");
+ pigServer.registerQuery("b = foreach a generate $0, flatten($1);");
+ Iterator<Tuple> iter = pigServer.openIterator("a");
+ int count = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ assertTrue(t.getAtomField(0).strval().equals("1"));
+ assertTrue(t.getAtomField(1).strval().equals("a"));
+ count++;
+ }
+ assertEquals(count, 6);
+ f.delete();
+ }
+
+
+ @Test
+ public void testMapLookup() throws Throwable {
DataBag b = BagFactory.getInstance().newDefaultBag();
- DataMap colors = new DataMap();
- colors.put("apple","red");
- colors.put("orange","orange");
-
- DataMap weights = new DataMap();
- weights.put("apple","0.1");
- weights.put("orange","0.3");
-
- Tuple t = new Tuple();
- t.appendField(colors);
- t.appendField(weights);
- b.add(t);
-
- String fileName = "file:"+File.createTempFile("tmp", "");
- PigFile f = new PigFile(fileName);
- f.store(b, new BinStorage(), pigServer.getPigContext());
-
-
- pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
- pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
- Iterator<Tuple> iter = pigServer.openIterator("b");
- t = iter.next();
- assertEquals(t.getAtomField(0).strval(), "red");
- assertEquals(t.getAtomField(1).numval(), 0.3);
- assertFalse(iter.hasNext());
- }
-
-
- static public class TitleNGrams extends EvalFunc<DataBag> {
-
- @Override
- public void exec(Tuple input, DataBag output) throws IOException {
- String str = input.getAtomField(0).strval();
-
- String title = str;
-
- if (title != null) {
- List<String> nGrams = makeNGrams(title);
-
- for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) {
- Tuple t = new Tuple(1);
- t.setField(0, it.next());
- output.add(t);
- }
- }
- }
-
-
- List<String> makeNGrams(String str) {
- List<String> tokens = new ArrayList<String>();
-
- StringTokenizer st = new StringTokenizer(str);
- while (st.hasMoreTokens())
- tokens.add(st.nextToken());
-
- return nGramHelper(tokens, new ArrayList<String>());
- }
-
- ArrayList<String> nGramHelper(List<String> str, ArrayList<String> nGrams) {
- if (str.size() == 0)
- return nGrams;
-
- for (int i = 0; i < str.size(); i++)
- nGrams.add(makeString(str.subList(0, i+1)));
-
- return nGramHelper(str.subList(1, str.size()), nGrams);
- }
-
- String makeString(List<String> list) {
- StringBuilder sb = new StringBuilder();
- for (Iterator<String> it = list.iterator(); it.hasNext(); ) {
- sb.append(it.next());
- if (it.hasNext())
- sb.append(" ");
- }
- return sb.toString();
- }
- }
-
- @Test
- public void testBagFunctionWithFlattening() throws Throwable {
- PigServer pigServer = new PigServer(MAPREDUCE);
- File queryLogFile = createFile(
- new String[]{
- "stanford\tdeer\tsighting",
- "bush\tpresident",
- "stanford\tbush",
- "conference\tyahoo",
- "world\tcup\tcricket",
- "bush\twins",
- "stanford\tpresident",
- }
- );
-
- File newsFile = createFile(
- new String[]{
- "deer seen at stanford",
- "george bush visits stanford",
- "yahoo hosting a conference in the bay area",
- "who will win the world cup"
- }
- );
-
- Map<String, Integer> expectedResults = new HashMap<String, Integer>();
- expectedResults.put("bush", 2);
- expectedResults.put("stanford", 3);
- expectedResults.put("world", 1);
- expectedResults.put("conference", 1);
-
- pigServer.registerQuery("newsArticles = LOAD 'file:" + newsFile + "' USING " + TextLoader.class.getName() + "();");
- pigServer.registerQuery("queryLog = LOAD 'file:" + queryLogFile + "';");
-
- pigServer.registerQuery("titleNGrams = FOREACH newsArticles GENERATE flatten(" + TitleNGrams.class.getName() + "(*));");
- pigServer.registerQuery("cogrouped = COGROUP titleNGrams BY $0 INNER, queryLog BY $0 INNER;");
- pigServer.registerQuery("answer = FOREACH cogrouped GENERATE COUNT(queryLog),group;");
-
- Iterator<Tuple> iter = pigServer.openIterator("answer");
- while(iter.hasNext()){
- Tuple t = iter.next();
- assertEquals(expectedResults.get(t.getAtomField(1).strval()).doubleValue(),t.getAtomField(0).numval().doubleValue());
- }
- }
-
- @Test
- public void testSort() throws Throwable {
- testSortDistinct(false);
- }
-
- @Test
- public void testDistinct() throws Throwable {
- testSortDistinct(true);
- }
-
- private void testSortDistinct(boolean eliminateDuplicates) throws Throwable {
- int LOOP_SIZE = 1024*16;
- File tmpFile = File.createTempFile("test", "txt");
- PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
- Random r = new Random();
- for(int i = 0; i < LOOP_SIZE; i++) {
- ps.println(r.nextInt(LOOP_SIZE/2) + "\t" + i);
- }
- ps.close();
-
- PigServer pig = new PigServer(MAPREDUCE);
- String tmpOutputFile = FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString();
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
- if (eliminateDuplicates){
- pig.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;");
- }else{
- pig.registerQuery("B = ORDER A BY $0 PARALLEL 10;");
- }
- pig.store("B", tmpOutputFile);
-
- pig.registerQuery("A = load '" + tmpOutputFile + "';");
- Iterator<Tuple> iter = pig.openIterator("A");
- int last = -1;
- while (iter.hasNext()){
- Tuple t = iter.next();
- if (eliminateDuplicates){
- assertTrue(last < t.getAtomField(0).numval().intValue());
- }else{
- assertTrue(last <= t.getAtomField(0).numval().intValue());
- assertEquals(t.arity(), 2);
- }
- }
-
- }
-
-
-}
+ DataMap colors = new DataMap();
+ colors.put("apple","red");
+ colors.put("orange","orange");
+
+ DataMap weights = new DataMap();
+ weights.put("apple","0.1");
+ weights.put("orange","0.3");
+
+ Tuple t = new Tuple();
+ t.appendField(colors);
+ t.appendField(weights);
+ b.add(t);
+
+ String fileName = "file:"+File.createTempFile("tmp", "");
+ PigFile f = new PigFile(fileName);
+ f.store(b, new BinStorage(), pigServer.getPigContext());
+
+
+ pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
+ pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
+ Iterator<Tuple> iter = pigServer.openIterator("b");
+ t = iter.next();
+ assertEquals(t.getAtomField(0).strval(), "red");
+ assertEquals(t.getAtomField(1).numval(), 0.3);
+ assertFalse(iter.hasNext());
+ }
+
+
+ static public class TitleNGrams extends EvalFunc<DataBag> {
+
+ @Override
+ public void exec(Tuple input, DataBag output) throws IOException {
+ String str = input.getAtomField(0).strval();
+
+ String title = str;
+
+ if (title != null) {
+ List<String> nGrams = makeNGrams(title);
+
+ for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) {
+ Tuple t = new Tuple(1);
+ t.setField(0, it.next());
+ output.add(t);
+ }
+ }
+ }
+
+
+ List<String> makeNGrams(String str) {
+ List<String> tokens = new ArrayList<String>();
+
+ StringTokenizer st = new StringTokenizer(str);
+ while (st.hasMoreTokens())
+ tokens.add(st.nextToken());
+
+ return nGramHelper(tokens, new ArrayList<String>());
+ }
+
+ ArrayList<String> nGramHelper(List<String> str, ArrayList<String> nGrams) {
+ if (str.size() == 0)
+ return nGrams;
+
+ for (int i = 0; i < str.size(); i++)
+ nGrams.add(makeString(str.subList(0, i+1)));
+
+ return nGramHelper(str.subList(1, str.size()), nGrams);
+ }
+
+ String makeString(List<String> list) {
+ StringBuilder sb = new StringBuilder();
+ for (Iterator<String> it = list.iterator(); it.hasNext(); ) {
+ sb.append(it.next());
+ if (it.hasNext())
+ sb.append(" ");
+ }
+ return sb.toString();
+ }
+ }
+
+ @Test
+ public void testBagFunctionWithFlattening() throws Throwable {
+ File queryLogFile = createFile(
+ new String[]{
+ "stanford\tdeer\tsighting",
+ "bush\tpresident",
+ "stanford\tbush",
+ "conference\tyahoo",
+ "world\tcup\tcricket",
+ "bush\twins",
+ "stanford\tpresident",
+ }
+ );
+
+ File newsFile = createFile(
+ new String[]{
+ "deer seen at stanford",
+ "george bush visits stanford",
+ "yahoo hosting a conference in the bay area",
+ "who will win the world cup"
+ }
+ );
+
+ Map<String, Integer> expectedResults = new HashMap<String, Integer>();
+ expectedResults.put("bush", 2);
+ expectedResults.put("stanford", 3);
+ expectedResults.put("world", 1);
+ expectedResults.put("conference", 1);
+
+ pigServer.registerQuery("newsArticles = LOAD 'file:" + newsFile + "' USING " + TextLoader.class.getName() + "();");
+ pigServer.registerQuery("queryLog = LOAD 'file:" + queryLogFile + "';");
+
+ pigServer.registerQuery("titleNGrams = FOREACH newsArticles GENERATE flatten(" + TitleNGrams.class.getName() + "(*));");
+ pigServer.registerQuery("cogrouped = COGROUP titleNGrams BY $0 INNER, queryLog BY $0 INNER;");
+ pigServer.registerQuery("answer = FOREACH cogrouped GENERATE COUNT(queryLog),group;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("answer");
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ assertEquals(expectedResults.get(t.getAtomField(1).strval()).doubleValue(),t.getAtomField(0).numval().doubleValue());
+ }
+ }
+
+ @Test
+ public void testSort() throws Throwable {
+ testSortDistinct(false);
+ }
+
+ @Test
+ public void testDistinct() throws Throwable {
+ testSortDistinct(true);
+ }
+
+ private void testSortDistinct(boolean eliminateDuplicates) throws Throwable {
+ int LOOP_SIZE = 1024*16;
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ Random r = new Random();
+ for(int i = 0; i < LOOP_SIZE; i++) {
+ ps.println(r.nextInt(LOOP_SIZE/2) + "\t" + i);
+ }
+ ps.close();
+
+ String tmpOutputFile = FileLocalizer.getTemporaryPath(null, pigServer.getPigContext()).toString();
+ pigServer.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ if (eliminateDuplicates){
+ pigServer.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;");
+ }else{
+ pigServer.registerQuery("B = ORDER A BY $0 PARALLEL 10;");
+ }
+ pigServer.store("B", tmpOutputFile);
+
+ pigServer.registerQuery("A = load '" + tmpOutputFile + "';");
+ Iterator<Tuple> iter = pigServer.openIterator("A");
+ int last = -1;
+ while (iter.hasNext()){
+ Tuple t = iter.next();
+ if (eliminateDuplicates){
+ assertTrue(last < t.getAtomField(0).numval().intValue());
+ }else{
+ assertTrue(last <= t.getAtomField(0).numval().intValue());
+ assertEquals(t.arity(), 2);
+ }
+ }
+
+ }
+
+
+}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java Thu Apr 10 15:14:04 2008
@@ -23,6 +23,7 @@
import java.io.PrintStream;
import java.util.Iterator;
+import org.junit.Before;
import org.junit.Test;
import org.apache.commons.logging.Log;
@@ -40,9 +41,16 @@
private static int LOOP_COUNT = 1024;
MiniCluster cluster = MiniCluster.buildCluster();
+ private PigServer pig;
+
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
+ }
+
@Test
public void testNumericEq() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -73,7 +81,6 @@
@Test
public void testNumericNeq() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -100,7 +107,6 @@
@Test
public void testNumericGt() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -128,7 +134,6 @@
@Test
public void testBinCond() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -157,7 +162,6 @@
@Test
public void testNestedBinCond() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -185,9 +189,8 @@
@Test
public void testNumericLt() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
- File tmpFile = File.createTempFile("test", "txt");
- PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
if(i % 5 == 0) {
ps.println(i + ":" + (double)i);
@@ -209,13 +212,11 @@
Double second = t.getAtomField(1).numval();
assertTrue(first.compareTo(second) < 0);
}
-
}
@Test
public void testNumericGte() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -245,7 +246,6 @@
@Test
public void testNumericLte() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java Thu Apr 10 15:14:04 2008
@@ -23,6 +23,7 @@
import java.io.PrintStream;
import java.util.Iterator;
+import org.junit.Before;
import org.junit.Test;
import org.apache.commons.logging.Log;
@@ -37,12 +38,19 @@
private final Log log = LogFactory.getLog(getClass());
- private static int LOOP_COUNT = 1024;
+ private static int LOOP_COUNT = 1024;
MiniCluster cluster = MiniCluster.buildCluster();
+ private PigServer pig;
+
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
+ }
+
@Test
public void testStringEq() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -70,7 +78,6 @@
@Test
public void testStringNeq() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -98,7 +105,6 @@
@Test
public void testStringGt() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -128,7 +134,6 @@
@Test
public void testStringGte() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -159,7 +164,6 @@
@Test
public void testStringLt() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -188,7 +192,6 @@
@Test
public void testStringLte() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Apr 10 15:14:04 2008
@@ -4,6 +4,7 @@
import junit.framework.TestCase;
import org.apache.pig.PigServer;
+import org.apache.pig.PigServer.ExecType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.grunt.Grunt;
@@ -17,7 +18,7 @@
@Test
public void testCopyFromLocal() throws Throwable {
- PigServer server = new PigServer("MAPREDUCE");
+ PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PigContext context = server.getPigContext();
String strCmd = "copyFromLocal /tmp/TestMe;";
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java Thu Apr 10 15:14:04 2008
@@ -24,6 +24,7 @@
import java.io.PrintStream;
import java.util.Iterator;
+import org.junit.Before;
import org.junit.Test;
import org.apache.commons.logging.Log;
@@ -40,10 +41,17 @@
private static int LOOP_COUNT = 1024;
MiniCluster cluster = MiniCluster.buildCluster();
+
+ private PigServer pig;
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
+ }
+
@Test
public void testAdd() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -65,7 +73,6 @@
@Test
public void testSubtract() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -86,7 +93,6 @@
@Test
public void testMultiply() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -108,7 +114,6 @@
@Test
public void testDivide() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 1; i < LOOP_COUNT; i++) {
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java Thu Apr 10 15:14:04 2008
@@ -89,7 +89,7 @@
dat.close();
try {
- pig = new PigServer(MAPREDUCE);
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
}
catch (ExecException e) {
IOException ioe = new IOException("Failed to create Pig server");
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu Apr 10 15:14:04 2008
@@ -21,6 +21,7 @@
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import junit.framework.AssertionFailedError;
@@ -455,7 +456,7 @@
@Test
public void testRegressionPig100NoSuchAlias() throws Throwable {
- PigContext pigContext = new PigContext(ExecType.LOCAL);
+ PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
boolean caughtIt = false;
@@ -472,7 +473,7 @@
@Test
public void testRegressionPig100NoAliases() throws Throwable {
- PigContext pigContext = new PigContext(ExecType.LOCAL);
+ PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
boolean caughtIt = false;
@@ -573,7 +574,7 @@
public LogicalPlan buildPlan(String query, ClassLoader cldr) {
LogicalPlanBuilder.classloader = cldr;
- PigContext pigContext = new PigContext(ExecType.LOCAL);
+ PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
try {
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java Thu Apr 10 15:14:04 2008
@@ -18,6 +18,7 @@
package org.apache.pig.test;
import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
@@ -27,33 +28,44 @@
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Iterator;
+import java.util.Properties;
import junit.framework.TestCase;
-import org.junit.Test;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.EvalFunc;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigServer;
import org.apache.pig.StoreFunc;
-import org.apache.pig.PigServer.ExecType;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
import org.apache.pig.builtin.COUNT;
import org.apache.pig.data.DataAtom;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.io.BufferedPositionedInputStream;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.backend.datastorage.ElementDescriptor;
+import org.apache.pig.impl.io.BufferedPositionedInputStream;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.junit.Before;
+import org.junit.Test;
public class TestMapReduce extends TestCase {
- MiniCluster cluster = MiniCluster.buildCluster();
+ private Log log = LogFactory.getLog(getClass());
+
+ MiniCluster cluster = MiniCluster.buildCluster();
- @Test
+ private PigServer pig;
+
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
+ }
+
+ @Test
public void testBigGroupAll() throws Throwable {
int LOOP_COUNT = 4*1024;
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
@@ -120,7 +132,6 @@
}
@Test
public void testStoreFunction() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < 10; i++) {
@@ -147,7 +158,6 @@
}
@Test
public void testQualifiedFuncions() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < 1; i++) {
@@ -172,7 +182,6 @@
@Test
public void testDefinedFunctions() throws Throwable {
- PigServer pig = new PigServer(MAPREDUCE);
File tmpFile = File.createTempFile("test", ".txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < 1; i++) {
@@ -198,8 +207,8 @@
@Test
public void testPigServer() throws Throwable {
- System.out.println("creating pig server");
- PigContext pigContext = new PigContext(MAPREDUCE);
+ log.debug("creating pig server");
+ PigContext pigContext = new PigContext(MAPREDUCE, cluster.getProperties());
PigServer pig = new PigServer(pigContext);
System.out.println("testing capacity");
long capacity = pig.capacity();
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestOrderBy.java Thu Apr 10 15:14:04 2008
@@ -1,218 +1,218 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-import java.text.DecimalFormat;
-import java.util.Iterator;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-
-import org.apache.pig.PigServer;
-import org.apache.pig.data.Tuple;
-import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
-
-public class TestOrderBy extends TestCase {
- private static final int DATALEN = 1024;
- private String[][] DATA = new String[2][DATALEN];
- MiniCluster cluster = MiniCluster.buildCluster();
-
- private PigServer pig;
- private File tmpFile;
-
- public TestOrderBy() throws Throwable {
- DecimalFormat myFormatter = new DecimalFormat("0000000");
- for (int i = 0; i < DATALEN; i++) {
- DATA[0][i] = myFormatter.format(i);
- DATA[1][i] = myFormatter.format(DATALEN - i - 1);
- }
- pig = new PigServer(MAPREDUCE);
- }
-
- protected void setUp() throws Exception {
- tmpFile = File.createTempFile("test", "txt");
- PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
- for(int i = 0; i < DATALEN; i++) {
- ps.println("1\t" + DATA[1][i] + "\t" + DATA[0][i]);
- }
- ps.close();
- }
-
- protected void tearDown() throws Exception {
- tmpFile.delete();
- }
-
- private void verify(String query, boolean descending) throws Exception {
- pig.registerQuery(query);
- Iterator<Tuple> it = pig.openIterator("myid");
- int col = (descending ? 1 : 0);
- for(int i = 0; i < DATALEN; i++) {
- Tuple t = (Tuple)it.next();
- int value = t.getAtomField(1).numval().intValue();
-// log.info("" + i + "," + DATA[0][i] + "," + DATA[1][i] + "," + value);
- assertEquals(Integer.parseInt(DATA[col][i]), value);
- }
- assertFalse(it.hasNext());
- }
-
- @Test
- public void testTopLevelOrderBy_Star_NoUsing() throws Exception {
- verify("myid = order (load 'file:" + tmpFile + "') BY *;", false);
- }
-
- @Test
- public void testTopLevelOrderBy_Col1_NoUsing() throws Exception {
- verify("myid = order (load 'file:" + tmpFile + "') BY $1;", false);
- }
-
- @Test
- public void testTopLevelOrderBy_Col2_NoUsing() throws Exception {
- verify("myid = order (load 'file:" + tmpFile + "') BY $2;", true);
- }
-
- @Test
- public void testTopLevelOrderBy_Col21_NoUsing() throws Exception {
- verify("myid = order (load 'file:" + tmpFile + "') BY $2, $1;", true);
- }
-
- @Test
- public void testTopLevelOrderBy_Star_Using() throws Exception {
- verify("myid = order (load 'file:" + tmpFile +
- "') BY * USING org.apache.pig.test.OrdAsc;", false);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY * USING org.apache.pig.test.OrdDesc;", true);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY * USING org.apache.pig.test.OrdDescNumeric;", true);
- }
-
- @Test
- public void testTopLevelOrderBy_Col1_Using() throws Exception {
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $1 USING org.apache.pig.test.OrdAsc;", false);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $1 USING org.apache.pig.test.OrdDesc;", true);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $1 USING org.apache.pig.test.OrdDescNumeric;", true);
- }
-
- @Test
- public void testTopLevelOrderBy_Col2_Using() throws Exception {
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2 USING org.apache.pig.test.OrdAsc;", true);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2 USING org.apache.pig.test.OrdDesc;", false);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2 USING org.apache.pig.test.OrdDescNumeric;", false);
- }
-
- @Test
- public void testTopLevelOrderBy_Col21_Using() throws Exception {
- // col2/col1 ascending -
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2, $1 USING org.apache.pig.test.OrdAsc;", true);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2, $1 USING org.apache.pig.test.OrdDesc;", false);
- verify("myid = order (load 'file:" + tmpFile +
- "') BY $2, $1 USING org.apache.pig.test.OrdDescNumeric;", false);
- }
-
- @Test
- public void testNestedOrderBy_Star_NoUsing() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY *; generate flatten(D); };", false);
- }
-
- @Test
- public void testNestedOrderBy_Col1_NoUsing() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $1; generate flatten(D); };", false);
- }
-
- @Test
- public void testNestedOrderBy_Col2_NoUsing() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2; generate flatten(D); };", true);
- }
-
- @Test
- public void testNestedOrderBy_Col21_NoUsing() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2, $1; generate flatten(D); };", true);
- }
-
- @Test
- public void testNestedOrderBy_Star_Using() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY * USING " +
- "org.apache.pig.test.OrdAsc; generate flatten(D); };", false);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY * USING " +
- "org.apache.pig.test.OrdDesc; generate flatten(D); };", true);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY * USING " +
- "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };", true);
- }
-
- @Test
- public void testNestedOrderBy_Col1_Using() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $1 USING " +
- "org.apache.pig.test.OrdAsc; generate flatten(D); };", false);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $1 USING " +
- "org.apache.pig.test.OrdDesc; generate flatten(D); };", true);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $1 USING " +
- "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
- true);
- }
-
- @Test
- public void testNestedOrderBy_Col2_Using() throws Exception {
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2 USING " +
- "org.apache.pig.test.OrdAsc; generate flatten(D); };", true);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2 USING " +
- "org.apache.pig.test.OrdDesc; generate flatten(D); };", false);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2 USING " +
- "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
- false);
- }
-
- @Test
- public void testNestedOrderBy_Col21_Using() throws Exception {
- // col2/col1 ascending -
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2, $1 USING " +
- "org.apache.pig.test.OrdAsc; generate flatten(D); };", true);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2, $1 USING " +
- "org.apache.pig.test.OrdDesc; generate flatten(D); };", false);
- verify("myid = foreach (group (load 'file:" + tmpFile +
- "') by $0) { D = ORDER $1 BY $2, $1 USING " +
- "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
- false);
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.text.DecimalFormat;
+import java.util.Iterator;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.data.Tuple;
+import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
+
+public class TestOrderBy extends TestCase {
+ private static final int DATALEN = 1024;
+ private String[][] DATA = new String[2][DATALEN];
+ MiniCluster cluster = MiniCluster.buildCluster();
+
+ private PigServer pig;
+ private File tmpFile;
+
+ public TestOrderBy() throws Throwable {
+ DecimalFormat myFormatter = new DecimalFormat("0000000");
+ for (int i = 0; i < DATALEN; i++) {
+ DATA[0][i] = myFormatter.format(i);
+ DATA[1][i] = myFormatter.format(DATALEN - i - 1);
+ }
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
+ }
+
+ protected void setUp() throws Exception {
+ tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ for(int i = 0; i < DATALEN; i++) {
+ ps.println("1\t" + DATA[1][i] + "\t" + DATA[0][i]);
+ }
+ ps.close();
+ }
+
+ protected void tearDown() throws Exception {
+ tmpFile.delete();
+ }
+
+ private void verify(String query, boolean descending) throws Exception {
+ pig.registerQuery(query);
+ Iterator<Tuple> it = pig.openIterator("myid");
+ int col = (descending ? 1 : 0);
+ for(int i = 0; i < DATALEN; i++) {
+ Tuple t = (Tuple)it.next();
+ int value = t.getAtomField(1).numval().intValue();
+// log.info("" + i + "," + DATA[0][i] + "," + DATA[1][i] + "," + value);
+ assertEquals(Integer.parseInt(DATA[col][i]), value);
+ }
+ assertFalse(it.hasNext());
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Star_NoUsing() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile + "') BY *;", false);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col1_NoUsing() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile + "') BY $1;", false);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col2_NoUsing() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile + "') BY $2;", true);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col21_NoUsing() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile + "') BY $2, $1;", true);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Star_Using() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY * USING org.apache.pig.test.OrdAsc;", false);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY * USING org.apache.pig.test.OrdDesc;", true);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY * USING org.apache.pig.test.OrdDescNumeric;", true);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col1_Using() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $1 USING org.apache.pig.test.OrdAsc;", false);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $1 USING org.apache.pig.test.OrdDesc;", true);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $1 USING org.apache.pig.test.OrdDescNumeric;", true);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col2_Using() throws Exception {
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2 USING org.apache.pig.test.OrdAsc;", true);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2 USING org.apache.pig.test.OrdDesc;", false);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2 USING org.apache.pig.test.OrdDescNumeric;", false);
+ }
+
+ @Test
+ public void testTopLevelOrderBy_Col21_Using() throws Exception {
+ // col2/col1 ascending -
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2, $1 USING org.apache.pig.test.OrdAsc;", true);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2, $1 USING org.apache.pig.test.OrdDesc;", false);
+ verify("myid = order (load 'file:" + tmpFile +
+ "') BY $2, $1 USING org.apache.pig.test.OrdDescNumeric;", false);
+ }
+
+ @Test
+ public void testNestedOrderBy_Star_NoUsing() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY *; generate flatten(D); };", false);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col1_NoUsing() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $1; generate flatten(D); };", false);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col2_NoUsing() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2; generate flatten(D); };", true);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col21_NoUsing() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2, $1; generate flatten(D); };", true);
+ }
+
+ @Test
+ public void testNestedOrderBy_Star_Using() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY * USING " +
+ "org.apache.pig.test.OrdAsc; generate flatten(D); };", false);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY * USING " +
+ "org.apache.pig.test.OrdDesc; generate flatten(D); };", true);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY * USING " +
+ "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };", true);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col1_Using() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $1 USING " +
+ "org.apache.pig.test.OrdAsc; generate flatten(D); };", false);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $1 USING " +
+ "org.apache.pig.test.OrdDesc; generate flatten(D); };", true);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $1 USING " +
+ "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
+ true);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col2_Using() throws Exception {
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2 USING " +
+ "org.apache.pig.test.OrdAsc; generate flatten(D); };", true);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2 USING " +
+ "org.apache.pig.test.OrdDesc; generate flatten(D); };", false);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2 USING " +
+ "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
+ false);
+ }
+
+ @Test
+ public void testNestedOrderBy_Col21_Using() throws Exception {
+ // col2/col1 ascending -
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2, $1 USING " +
+ "org.apache.pig.test.OrdAsc; generate flatten(D); };", true);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2, $1 USING " +
+ "org.apache.pig.test.OrdDesc; generate flatten(D); };", false);
+ verify("myid = foreach (group (load 'file:" + tmpFile +
+ "') by $0) { D = ORDER $1 BY $2, $1 USING " +
+ "org.apache.pig.test.OrdDescNumeric; generate flatten(D); };",
+ false);
+ }
+
+}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPi.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPi.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPi.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPi.java Thu Apr 10 15:14:04 2008
@@ -99,7 +99,7 @@
dat.close();
try {
- pig = new PigServer(MAPREDUCE);
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
}
catch (ExecException e) {
IOException ioe = new IOException("Failed to create Pig Server");
Added: incubator/pig/trunk/test/org/apache/pig/test/TestPigContext.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=646988&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigContext.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigContext.java Thu Apr 10 15:14:04 2008
@@ -0,0 +1,106 @@
+package org.apache.pig.test;
+
+import static org.apache.pig.PigServer.ExecType.LOCAL;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestPigContext extends TestCase {
+
+ private static final String TMP_DIR_PROP = "/tmp/hadoop-hadoop";
+ private static final String FS_NAME = "machine:9000";
+ private static final String JOB_TRACKER = "machine:9001";
+
+ private File input;
+ private PigContext pigContext;
+
+ @Before
+ @Override
+ protected void setUp() throws Exception {
+ pigContext = new PigContext(LOCAL, getProperties());
+ input = File.createTempFile("PigContextTest-", ".txt");
+ }
+
+ /**
+ * Passing an already configured pigContext in PigServer constructor.
+ */
+ @Test
+ public void testSetProperties_way_num01() throws Exception {
+ PigServer pigServer = new PigServer(pigContext);
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ /**
+ * Setting properties through PigServer constructor directly.
+ */
+ @Test
+ public void testSetProperties_way_num02() throws Exception {
+ PigServer pigServer = new PigServer(LOCAL, getProperties());
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ /**
+ * using connect() method.
+ */
+ @Test
+ public void testSetProperties_way_num03() throws Exception {
+ pigContext.connect();
+ PigServer pigServer = new PigServer(pigContext);
+ registerAndStore(pigServer);
+
+ check_asserts();
+ }
+
+ @After
+ @Override
+ protected void tearDown() throws Exception {
+ input.delete();
+ }
+
+ private static Properties getProperties() {
+ Properties props = new Properties();
+ props.put("mapred.job.tracker", JOB_TRACKER);
+ props.put("fs.default.name", FS_NAME);
+ props.put("hadoop.tmp.dir", TMP_DIR_PROP);
+ return props;
+ }
+
+ private List<String> getCommands() {
+ List<String> commands = new ArrayList<String>();
+ commands.add("my_input = LOAD '" + input.getAbsolutePath() + "' USING PigStorage();");
+ commands.add("words = FOREACH my_input GENERATE FLATTEN(TOKENIZE(*));");
+ commands.add("grouped = GROUP words BY $0;");
+ commands.add("counts = FOREACH grouped GENERATE group, COUNT(words);");
+ return commands;
+ }
+
+ private void registerAndStore(PigServer pigServer) throws IOException {
+ pigServer.debugOn();
+ List<String> commands = getCommands();
+ for (final String command : commands) {
+ pigServer.registerQuery(command);
+ }
+ pigServer.store("counts", input.getAbsolutePath() + ".out");
+ }
+
+ private void check_asserts() {
+ assertEquals(JOB_TRACKER, pigContext.getProperties().getProperty("mapred.job.tracker"));
+ assertEquals(FS_NAME, pigContext.getProperties().getProperty("fs.default.name"));
+ assertEquals(TMP_DIR_PROP, pigContext.getProperties().getProperty("hadoop.tmp.dir"));
+ }
+}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java Thu Apr 10 15:14:04 2008
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.PrintStream;
import java.util.Iterator;
+import java.util.Properties;
import java.util.Random;
import junit.framework.TestCase;
@@ -76,7 +77,7 @@
@Test
public void testStoreAndLoadText() throws IOException {
- PigContext pigContext = new PigContext(ExecType.LOCAL);
+ PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
log.info("Running Store...");
String initialdata = File.createTempFile("pig-tmp", "").getAbsolutePath();
@@ -155,7 +156,7 @@
bag = getRandomBag(5000,0);
log.info("Done.");
- PigContext pigContext = new PigContext(ExecType.LOCAL);
+ PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
log.info("Running Store...");
String storeFile = File.createTempFile("pig-tmp", "").getAbsolutePath();
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Thu Apr 10 15:14:04 2008
@@ -7,6 +7,7 @@
import java.io.PrintStream;
import java.util.Map;
import java.util.HashMap;
+import java.util.Properties;
import org.junit.Test;
import junit.framework.TestCase;
@@ -26,7 +27,7 @@
// All the needed variables
Map<String, LogicalPlan> aliases = new HashMap<String, LogicalPlan>() ;
Map<OperatorKey, LogicalOperator> opTable = new HashMap<OperatorKey, LogicalOperator>() ;
- PigContext pigContext = new PigContext(PigServer.ExecType.LOCAL) ;
+ PigContext pigContext = new PigContext(PigServer.ExecType.LOCAL, new Properties()) ;
String tempFile = this.prepareTempFile() ;
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigServer.java Thu Apr 10 15:14:04 2008
@@ -1,250 +1,251 @@
-
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
-import java.util.List;
-import java.util.Iterator;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.lang.reflect.Method;
-
-import org.apache.pig.PigServer;
-
-import org.junit.Test;
-import junit.framework.TestCase;
-
-
-public class TestPigServer extends TestCase {
- private PigServer pig = null;
- MiniCluster cluster = MiniCluster.buildCluster();
-
- private void initPigServer() throws Throwable {
- if (pig == null) {
- pig = new PigServer();
- }
- }
-
-
- private final static String FILE_SEPARATOR = System.getProperty("file.separator");
-
- // make sure that name is included or not (depending on flag "included")
- // in the given list of stings
- private static void verifyStringContained(List<URL> list, String name, boolean included) {
- Iterator<URL> iter = list.iterator();
- boolean nameIsSubstring = false;
- int count = 0;
-
- while (iter.hasNext()) {
- if (iter.next().toString().contains(name)) {
- nameIsSubstring = true;
- ++count;
- }
- }
-
- if (included) {
- assertTrue(nameIsSubstring);
- assertTrue(count == 1);
- }
- else {
- assertFalse(nameIsSubstring);
- }
- }
-
- // creates an empty jar file
- private static void createFakeJarFile(String location, String name)
- throws IOException {
- assertFalse((new File(name)).canRead());
-
- assertTrue((new File(location)).mkdirs());
-
- assertTrue((new File(location + FILE_SEPARATOR + name)).
- createNewFile());
- }
-
- // dynamically add more resources to the system class loader
- private static void registerNewResource(String file) throws Exception {
- URL urlToAdd = new File(file).toURI().toURL();
- URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
- Method addMethod = URLClassLoader.class.
- getDeclaredMethod("addURL",
- new Class[]{URL.class});
- addMethod.setAccessible(true);
- addMethod.invoke(sysLoader, new Object[]{urlToAdd});
- }
-
- private static void executeShellCommand(String cmd) throws Exception {
- Process cmdProc = Runtime.getRuntime().exec(cmd);
-
- cmdProc.waitFor();
-
- assertTrue(cmdProc.exitValue() == 0);
- }
-
- /**
- * The jar file to register is not present
- */
- @Test
- public void testRegisterJarFileNotPresent() throws Throwable {
- // resister a jar file that does not exist
-
- String jarName = "BadFileNameTestJarNotPresent.jar";
-
- // jar name is not present to start with
- initPigServer();
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertTrue(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
- }
-
- /**
- * Jar file to register is not present in the system resources
- * in this case name of jar file is relative to current working dir
- */
- @Test
- public void testRegisterJarLocalDir() throws Throwable {
- String dir1 = "test1_register_jar_local";
- String dir2 = "test2_register_jar_local";
- String jarLocation = dir1 + FILE_SEPARATOR +
- dir2 + FILE_SEPARATOR;
- String jarName = "TestRegisterJarLocal.jar";
-
- initPigServer();
-
- createFakeJarFile(jarLocation, jarName);
-
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarLocation + jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean-up
- assertTrue((new File(jarLocation + jarName)).delete());
- (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
- (new File(dir1)).delete();
- }
-
- /**
- * Jar file is located via system resources
- * Test verifies that even with multiple resources matching,
- * only one of them is registered.
- */
- @Test
- public void testRegisterJarFromResources () throws Throwable {
- String dir = "test_register_jar_res_dir";
- String subDir1 = "test_register_jar_res_sub_dir1";
- String subDir2 = "test_register_jar_res_sub_dir2";
- String jarName = "TestRegisterJarFromRes.jar";
- String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
- String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
-
- initPigServer();
-
- createFakeJarFile(jarLocation1, jarName);
- createFakeJarFile(jarLocation2, jarName);
-
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- registerNewResource(jarLocation1);
- registerNewResource(jarLocation2);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean-up
- assertTrue((new File(jarLocation1 + jarName)).delete());
- assertTrue((new File(jarLocation2 + jarName)).delete());
- (new File(jarLocation1)).delete();
- (new File(jarLocation2)).delete();
- (new File(dir)).delete();
- }
-
- /**
- * Use a resource inside a jar file.
- * Verify that the containing jar file is registered correctly.
- * @throws Exception
- */
- @Test
- public void testRegisterJarResourceInJar() throws Throwable {
- String dir = "test_register_jar_res_in_jar";
- String subDir = "sub_dir";
- String jarName = "TestRegisterJarNonEmpty.jar";
- String className = "TestRegisterJar";
- String javaSrc = "package " + subDir + "; class " + className + " { }";
-
- initPigServer();
-
- // create dirs
- (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
-
- // generate java file
- FileOutputStream outStream =
- new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java"));
-
- OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
- outWriter.write(javaSrc);
- outWriter.close();
-
- // compile
- executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java");
-
- // remove src file
- (new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java")).delete();
-
- // generate jar file
- executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
- "-C " + dir + " " + subDir);
-
- // remove class file and sub_dir
- (new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".class")).delete();
- (new File(dir + FILE_SEPARATOR + subDir)).delete();
-
- // register resource
- registerNewResource(dir + FILE_SEPARATOR + jarName);
-
- // load the specific resource
- boolean exceptionRaised = false;
- try {
- pig.registerJar("sub_dir/TestRegisterJar.class");
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
-
- // verify proper jar file is located
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean up Jar file and test dir
- (new File(dir + FILE_SEPARATOR + jarName)).delete();
- (new File(dir)).delete();
- }
+
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.util.List;
+import java.util.Iterator;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.lang.reflect.Method;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.PigServer.ExecType;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+
+public class TestPigServer extends TestCase {
+ private PigServer pig = null;
+ MiniCluster cluster = MiniCluster.buildCluster();
+
+ private void initPigServer() throws Throwable {
+ if (pig == null) {
+ pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ }
+ }
+
+
+ private final static String FILE_SEPARATOR = System.getProperty("file.separator");
+
+ // make sure that name is included or not (depending on flag "included")
+ // in the given list of stings
+ private static void verifyStringContained(List<URL> list, String name, boolean included) {
+ Iterator<URL> iter = list.iterator();
+ boolean nameIsSubstring = false;
+ int count = 0;
+
+ while (iter.hasNext()) {
+ if (iter.next().toString().contains(name)) {
+ nameIsSubstring = true;
+ ++count;
+ }
+ }
+
+ if (included) {
+ assertTrue(nameIsSubstring);
+ assertTrue(count == 1);
+ }
+ else {
+ assertFalse(nameIsSubstring);
+ }
+ }
+
+ // creates an empty jar file
+ private static void createFakeJarFile(String location, String name)
+ throws IOException {
+ assertFalse((new File(name)).canRead());
+
+ assertTrue((new File(location)).mkdirs());
+
+ assertTrue((new File(location + FILE_SEPARATOR + name)).
+ createNewFile());
+ }
+
+ // dynamically add more resources to the system class loader
+ private static void registerNewResource(String file) throws Exception {
+ URL urlToAdd = new File(file).toURI().toURL();
+ URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+ Method addMethod = URLClassLoader.class.
+ getDeclaredMethod("addURL",
+ new Class[]{URL.class});
+ addMethod.setAccessible(true);
+ addMethod.invoke(sysLoader, new Object[]{urlToAdd});
+ }
+
+ private static void executeShellCommand(String cmd) throws Exception {
+ Process cmdProc = Runtime.getRuntime().exec(cmd);
+
+ cmdProc.waitFor();
+
+ assertTrue(cmdProc.exitValue() == 0);
+ }
+
+ /**
+ * The jar file to register is not present
+ */
+ @Test
+ public void testRegisterJarFileNotPresent() throws Throwable {
+ // resister a jar file that does not exist
+
+ String jarName = "BadFileNameTestJarNotPresent.jar";
+
+ // jar name is not present to start with
+ initPigServer();
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertTrue(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+ }
+
+ /**
+ * Jar file to register is not present in the system resources
+ * in this case name of jar file is relative to current working dir
+ */
+ @Test
+ public void testRegisterJarLocalDir() throws Throwable {
+ String dir1 = "test1_register_jar_local";
+ String dir2 = "test2_register_jar_local";
+ String jarLocation = dir1 + FILE_SEPARATOR +
+ dir2 + FILE_SEPARATOR;
+ String jarName = "TestRegisterJarLocal.jar";
+
+ initPigServer();
+
+ createFakeJarFile(jarLocation, jarName);
+
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarLocation + jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean-up
+ assertTrue((new File(jarLocation + jarName)).delete());
+ (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
+ (new File(dir1)).delete();
+ }
+
+ /**
+ * Jar file is located via system resources
+ * Test verifies that even with multiple resources matching,
+ * only one of them is registered.
+ */
+ @Test
+ public void testRegisterJarFromResources () throws Throwable {
+ String dir = "test_register_jar_res_dir";
+ String subDir1 = "test_register_jar_res_sub_dir1";
+ String subDir2 = "test_register_jar_res_sub_dir2";
+ String jarName = "TestRegisterJarFromRes.jar";
+ String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
+ String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
+
+ initPigServer();
+
+ createFakeJarFile(jarLocation1, jarName);
+ createFakeJarFile(jarLocation2, jarName);
+
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ registerNewResource(jarLocation1);
+ registerNewResource(jarLocation2);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean-up
+ assertTrue((new File(jarLocation1 + jarName)).delete());
+ assertTrue((new File(jarLocation2 + jarName)).delete());
+ (new File(jarLocation1)).delete();
+ (new File(jarLocation2)).delete();
+ (new File(dir)).delete();
+ }
+
+ /**
+ * Use a resource inside a jar file.
+ * Verify that the containing jar file is registered correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testRegisterJarResourceInJar() throws Throwable {
+ String dir = "test_register_jar_res_in_jar";
+ String subDir = "sub_dir";
+ String jarName = "TestRegisterJarNonEmpty.jar";
+ String className = "TestRegisterJar";
+ String javaSrc = "package " + subDir + "; class " + className + " { }";
+
+ initPigServer();
+
+ // create dirs
+ (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
+
+ // generate java file
+ FileOutputStream outStream =
+ new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java"));
+
+ OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
+ outWriter.write(javaSrc);
+ outWriter.close();
+
+ // compile
+ executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java");
+
+ // remove src file
+ (new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java")).delete();
+
+ // generate jar file
+ executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
+ "-C " + dir + " " + subDir);
+
+ // remove class file and sub_dir
+ (new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".class")).delete();
+ (new File(dir + FILE_SEPARATOR + subDir)).delete();
+
+ // register resource
+ registerNewResource(dir + FILE_SEPARATOR + jarName);
+
+ // load the specific resource
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar("sub_dir/TestRegisterJar.class");
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+
+ // verify proper jar file is located
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean up Jar file and test dir
+ (new File(dir + FILE_SEPARATOR + jarName)).delete();
+ (new File(dir)).delete();
+ }
}
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java Thu Apr 10 15:14:04 2008
@@ -18,13 +18,14 @@
package org.apache.pig.test;
-import java.io.BufferedWriter;
+import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
+
import java.io.File;
import java.io.IOException;
-import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Iterator;
+import org.junit.Before;
import org.junit.Test;
import org.apache.pig.PigServer;
@@ -38,17 +39,11 @@
MiniCluster cluster = MiniCluster.buildCluster();
@Override
+ @Before
protected void setUp() throws Exception {
super.setUp();
- try {
- pig = new PigServer();
- }
- catch (ExecException e) {
- IOException ioe = new IOException("Failed to create Pig Server");
- ioe.initCause(e);
- throw ioe;
- }
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
}
@Test
public void testLongEvalSpec() throws Exception{
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=646988&r1=646987&r2=646988&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestStore.java Thu Apr 10 15:14:04 2008
@@ -15,111 +15,111 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.test;
-
-import java.io.File;
+package org.apache.pig.test;
+
+import java.io.File;
import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Iterator;
-
-import org.apache.pig.PigServer;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.io.FileLocalizer;
+import java.io.PrintWriter;
+import java.util.Iterator;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.backend.executionengine.ExecException;
import static org.apache.pig.PigServer.ExecType.MAPREDUCE;
-
-import junit.framework.TestCase;
-
-public class TestStore extends TestCase {
-
- private int LOOP_COUNT = 1024;
+
+import junit.framework.TestCase;
+
+public class TestStore extends TestCase {
+
+ private int LOOP_COUNT = 1024;
MiniCluster cluster = MiniCluster.buildCluster();
-
- String fileName;
- String tmpFile1, tmpFile2;
- PigServer pig;
-
- public void testSingleStore() throws Exception{
- pig.registerQuery("A = load " + fileName + ";");
-
- pig.store("A", tmpFile1);
-
- pig.registerQuery("B = load " + tmpFile1 + ";");
- Iterator<Tuple> iter = pig.openIterator("B");
-
- int i =0;
- while (iter.hasNext()){
- Tuple t = iter.next();
- assertEquals(t.getAtomField(0).numval().intValue(),i);
- assertEquals(t.getAtomField(1).numval().intValue(),i);
- i++;
- }
- }
-
- public void testMultipleStore() throws Exception{
- pig.registerQuery("A = load " + fileName + ";");
-
- pig.store("A", tmpFile1);
-
- pig.registerQuery("B = foreach (group A by $0) generate $0, SUM($1);");
- pig.store("B", tmpFile2);
- pig.registerQuery("C = load " + tmpFile2 + ";");
- Iterator<Tuple> iter = pig.openIterator("C");
-
- int i =0;
- while (iter.hasNext()){
- Tuple t = iter.next();
+
+ String fileName;
+ String tmpFile1, tmpFile2;
+ PigServer pig;
+
+ public void testSingleStore() throws Exception{
+ pig.registerQuery("A = load " + fileName + ";");
+
+ pig.store("A", tmpFile1);
+
+ pig.registerQuery("B = load " + tmpFile1 + ";");
+ Iterator<Tuple> iter = pig.openIterator("B");
+
+ int i =0;
+ while (iter.hasNext()){
+ Tuple t = iter.next();
+ assertEquals(t.getAtomField(0).numval().intValue(),i);
+ assertEquals(t.getAtomField(1).numval().intValue(),i);
+ i++;
+ }
+ }
+
+ public void testMultipleStore() throws Exception{
+ pig.registerQuery("A = load " + fileName + ";");
+
+ pig.store("A", tmpFile1);
+
+ pig.registerQuery("B = foreach (group A by $0) generate $0, SUM($1);");
+ pig.store("B", tmpFile2);
+ pig.registerQuery("C = load " + tmpFile2 + ";");
+ Iterator<Tuple> iter = pig.openIterator("C");
+
+ int i =0;
+ while (iter.hasNext()){
+ Tuple t = iter.next();
+ i++;
+
+ }
+
+ assertEquals(LOOP_COUNT, i);
+
+ }
+
+ public void testStoreWithMultipleMRJobs() throws Exception{
+ pig.registerQuery("A = load " + fileName + ";");
+ pig.registerQuery("B = foreach (group A by $0) generate $0, SUM($1);");
+ pig.registerQuery("C = foreach (group B by $0) generate $0, SUM($1);");
+ pig.registerQuery("D = foreach (group C by $0) generate $0, SUM($1);");
+
+ pig.store("D", tmpFile2);
+ pig.registerQuery("E = load " + tmpFile2 + ";");
+ Iterator<Tuple> iter = pig.openIterator("E");
+
+ int i =0;
+ while (iter.hasNext()){
+ Tuple t = iter.next();
i++;
-
}
- assertEquals(LOOP_COUNT, i);
-
- }
-
- public void testStoreWithMultipleMRJobs() throws Exception{
- pig.registerQuery("A = load " + fileName + ";");
- pig.registerQuery("B = foreach (group A by $0) generate $0, SUM($1);");
- pig.registerQuery("C = foreach (group B by $0) generate $0, SUM($1);");
- pig.registerQuery("D = foreach (group C by $0) generate $0, SUM($1);");
-
- pig.store("D", tmpFile2);
- pig.registerQuery("E = load " + tmpFile2 + ";");
- Iterator<Tuple> iter = pig.openIterator("E");
-
- int i =0;
- while (iter.hasNext()){
- Tuple t = iter.next();
- i++;
- }
-
- assertEquals(LOOP_COUNT, i);
-
- }
-
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- File f = File.createTempFile("tmp", "");
- PrintWriter pw = new PrintWriter(f);
- for (int i=0;i<LOOP_COUNT; i++){
- pw.println(i + "\t" + i);
- }
+ assertEquals(LOOP_COUNT, i);
+
+ }
+
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ File f = File.createTempFile("tmp", "");
+ PrintWriter pw = new PrintWriter(f);
+ for (int i=0;i<LOOP_COUNT; i++){
+ pw.println(i + "\t" + i);
+ }
pw.close();
- try {
- pig = new PigServer(MAPREDUCE);
+ try {
+ pig = new PigServer(MAPREDUCE, cluster.getProperties());
}
catch (ExecException e) {
IOException ioe = new IOException("Failed to create Pig Server");
ioe.initCause(e);
throw ioe;
}
-
- fileName = "'" + FileLocalizer.hadoopify(f.toString(), pig.getPigContext()) + "'";
- tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
- tmpFile2 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
- f.delete();
- }
-
-}
+
+ fileName = "'" + FileLocalizer.hadoopify(f.toString(), pig.getPigContext()) + "'";
+ tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
+ tmpFile2 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
+ f.delete();
+ }
+
+}