You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/03/09 17:17:43 UTC
svn commit: r920956 [3/3] - in /hadoop/pig/trunk/contrib/zebra: ./
src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/mapreduce/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java?rev=920956&r1=920955&r2=920956&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java Tue Mar 9 16:17:42 2010
@@ -33,6 +33,7 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -48,6 +49,8 @@ import org.apache.hadoop.mapreduce.Mappe
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
@@ -88,7 +91,7 @@ import org.apache.hadoop.zebra.mapreduce
*
*
*/
-public class TestTypedApi {
+public class TestTypedApi extends Configured implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
protected static ExecType execType = ExecType.LOCAL;
@@ -112,40 +115,42 @@ public class TestTypedApi {
public static void setUpOnce() throws IOException {
if (System.getenv("hadoop.log.dir") == null) {
String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
}
- if (System.getProperty("whichCluster") == null) {
- System.setProperty("whichCluster", "miniCluster");
- System.out.println("should be called");
- whichCluster = System.getProperty("whichCluster");
- } else {
- whichCluster = System.getProperty("whichCluster");
- }
-
- System.out.println("clusterddddd: " + whichCluster);
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
- if ((whichCluster.equalsIgnoreCase("realCluster") && System
- .getenv("HADOOP_HOME") == null)) {
- System.out.println("Please set HADOOP_HOME");
- System.exit(0);
- }
-
- conf = new Configuration();
-
- if ((whichCluster.equalsIgnoreCase("realCluster") && System.getenv("USER") == null)) {
- System.out.println("Please set USER");
- System.exit(0);
- }
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put zebra.jar at hadoop_home/lib");
- System.exit(0);
- }
+ // by default we use miniCluster
+ if (System.getenv("whichCluster") == null) {
+ whichCluster = "miniCluster";
+ } else {
+ whichCluster = System.getenv("whichCluster");
+ }
+
+ if (conf == null) {
+ conf = new Configuration();
+ }
+
+ if (whichCluster.equals("realCluster")) {
+ System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
+ System.out.println(" get env user name: " + System.getenv("USER"));
+
+ if (System.getenv("HADOOP_HOME") == null) {
+ System.out.println("Please set HADOOP_HOME for realCluster testing mode");
+ System.exit(0);
+ }
+
+ if (System.getenv("USER") == null) {
+ System.out.println("Please set USER for realCluster testing mode");
+ System.exit(0);
+ }
+
+ zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+
+ File file = new File(zebraJar);
+ if (!file.exists()) {
+ System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
+ System.exit(0);
+ }
+ }
// set inputPath and output path
String workingDir = null;
@@ -520,7 +525,6 @@ public class TestTypedApi {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
checkTable(myMultiLocs);
System.out.println("DONE test " + getCurrentMethodName());
-
}
@Test(expected = ParseException.class)
@@ -562,8 +566,21 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "{, count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
-
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 2");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 2");
+ }
}
@Test(expected = IOException.class)
@@ -605,8 +622,21 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
-
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (IOException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 3");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 3");
+ }
}
@Test(expected = IOException.class)
@@ -615,7 +645,7 @@ public class TestTypedApi {
/*
* test negative test case. sort key is empty string
*/
- System.out.println("******Starttt testcase: " + getCurrentMethodName());
+ System.out.println("******Start testcase: " + getCurrentMethodName());
List<Path> paths = new ArrayList<Path>(1);
sortKey = "";
@@ -648,8 +678,21 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
-
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (IOException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 4");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 4");
+ }
}
@Test(expected = NullPointerException.class)
@@ -691,8 +734,21 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (NullPointerException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 5");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 5");
+ }
}
@Test(expected = ParseException.class)
@@ -734,8 +790,22 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[none-exist-column]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ //runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 6");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 6");
+ }
}
@Test(expected = ParseException.class)
@@ -778,8 +848,22 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "none-exist-column]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ //runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 7");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 7");
+ }
}
@Test(expected = ParseException.class)
@@ -822,8 +906,23 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:int,word:string, count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
-
+ //runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 8");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 8");
+ }
}
@Test(expected = ParseException.class)
@@ -866,7 +965,23 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string, count:inttt";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ //runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 9");
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 9");
+ }
}
@@ -910,8 +1025,24 @@ public class TestTypedApi {
removeDir(new Path(strTable2));
String schema = "word:string; count:int";
String storageHint = "[word];[count]";
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- System.out.println("done test 10");
+ //runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+
+ if (whichCluster.equals("realCluster")) {
+ try {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ return;
+ }
+
+ // should not reach here
+ Assert.fail("in try, should have thrown exception");
+ System.out.println("done test 10");
+
+ } else {
+ runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
+ System.out.println("done test 10");
+ }
}
static class MapClass extends
@@ -1047,8 +1178,9 @@ public class TestTypedApi {
Path... paths) throws ParseException, IOException, Exception,
org.apache.hadoop.zebra.parser.ParseException {
- Job job = new Job();
+ Job job = new Job(conf);
job.setJobName("TestTypedAPI");
+ job.setJarByClass(TestTypedApi.class);
Configuration conf = job.getConfiguration();
conf.set("table.output.tfile.compression", "gz");
conf.set("sortKey", sortKey);
@@ -1079,19 +1211,35 @@ public class TestTypedApi {
BasicTableOutputFormat.close( job );
}
- public static void main(String[] args) throws ParseException,
- org.apache.hadoop.zebra.parser.ParseException, Exception {
+ @Override
+ public int run(String[] args) throws Exception {
TestTypedApi test = new TestTypedApi();
TestTypedApi.setUpOnce();
test.test1();
test.test2();
test.test3();
- test.test4();
+
+ // backend exception - will migrate later
+ //test.test4();
+
test.test5();
test.test6();
test.test7();
test.test8();
test.test9();
test.test10();
+
+ return 0;
}
+
+ public static void main(String[] args) throws Exception {
+ //XXX
+ System.out.println("******************* this is new today");
+
+ conf = new Configuration();
+
+ int res = ToolRunner.run(conf, new TestTypedApi(), args);
+
+ System.exit(res);
+ }
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java?rev=920956&r1=920955&r2=920956&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java Tue Mar 9 16:17:42 2010
@@ -28,13 +28,11 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
-
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -45,19 +43,18 @@ import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.file.tfile.RawComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
-import org.apache.hadoop.zebra.mapreduce.ZebraProjection;
import org.apache.hadoop.zebra.mapreduce.ZebraSchema;
import org.apache.hadoop.zebra.mapreduce.ZebraSortInfo;
import org.apache.hadoop.zebra.mapreduce.ZebraStorageHint;
-import org.apache.hadoop.zebra.mapreduce.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
@@ -66,14 +63,12 @@ import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DefaultTuple;
+
import org.apache.pig.data.Tuple;
import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
-
/**
* This is a sample a complete MR sample code for Table. It doens't contain
* 'read' part. But, it should be similar and easier to write. Refer to test
@@ -93,7 +88,7 @@ import org.junit.Test;
*
*
*/
-public class TestTypedApi2 {
+public class TestTypedApi2 extends Configured implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
protected static ExecType execType = ExecType.LOCAL;
@@ -117,40 +112,42 @@ public class TestTypedApi2 {
public static void setUpOnce() throws IOException {
if (System.getenv("hadoop.log.dir") == null) {
String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- if (System.getProperty("whichCluster") == null) {
- System.setProperty("whichCluster", "miniCluster");
- System.out.println("should be called");
- whichCluster = System.getProperty("whichCluster");
- } else {
- whichCluster = System.getProperty("whichCluster");
- }
-
- System.out.println("clusterddddd: " + whichCluster);
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
- if ((whichCluster.equalsIgnoreCase("realCluster") && System
- .getenv("HADOOP_HOME") == null)) {
- System.out.println("Please set HADOOP_HOME");
- System.exit(0);
+ System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
}
- conf = new Configuration();
-
- if ((whichCluster.equalsIgnoreCase("realCluster") && System.getenv("USER") == null)) {
- System.out.println("Please set USER");
- System.exit(0);
- }
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put zebra.jar at hadoop_home/lib");
- System.exit(0);
- }
+ // by default we use miniCluster
+ if (System.getenv("whichCluster") == null) {
+ whichCluster = "miniCluster";
+ } else {
+ whichCluster = System.getenv("whichCluster");
+ }
+
+ if (conf == null) {
+ conf = new Configuration();
+ }
+
+ if (whichCluster.equals("realCluster")) {
+ System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
+ System.out.println(" get env user name: " + System.getenv("USER"));
+
+ if (System.getenv("HADOOP_HOME") == null) {
+ System.out.println("Please set HADOOP_HOME for realCluster testing mode");
+ System.exit(0);
+ }
+
+ if (System.getenv("USER") == null) {
+ System.out.println("Please set USER for realCluster testing mode");
+ System.exit(0);
+ }
+
+ zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+
+ File file = new File(zebraJar);
+ if (!file.exists()) {
+ System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
+ System.exit(0);
+ }
+ }
// set inputPath and output path
String workingDir = null;
@@ -626,7 +623,6 @@ public class TestTypedApi2 {
else
return 1;
}
-
}
public static final class MemcmpRawComparator implements
@@ -647,8 +643,9 @@ public class TestTypedApi2 {
Path... paths) throws ParseException, IOException, Exception,
org.apache.hadoop.zebra.parser.ParseException {
- Job job = new Job();
- job.setJobName("TestTypedAPI");
+ Job job = new Job(conf);
+ job.setJobName("TestTypedApi2");
+ job.setJarByClass(TestTypedApi2.class);
Configuration conf = job.getConfiguration();
conf.set("table.output.tfile.compression", "gz");
conf.set("sortKey", sortKey);
@@ -683,11 +680,23 @@ public class TestTypedApi2 {
BasicTableOutputFormat.close( job );
}
- public static void main(String[] args) throws ParseException,
- org.apache.hadoop.zebra.parser.ParseException, Exception {
+ @Override
+ public int run(String[] args) throws Exception {
TestTypedApi2 test = new TestTypedApi2();
TestTypedApi2.setUpOnce();
test.test1();
+ return 0;
}
+
+ public static void main(String[] args) throws Exception {
+ //XXX
+ System.out.println("******************* this is new today");
+
+ conf = new Configuration();
+
+ int res = ToolRunner.run(conf, new TestTypedApi2(), args);
+
+ System.exit(res);
+ }
}