You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/03/22 08:54:53 UTC
svn commit: r925988 [2/8] - in /hadoop/pig/trunk/contrib/zebra: ./
src/test/org/apache/hadoop/zebra/ src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/mapreduce/
src/test/org/apache/hadoop/zebra/pig/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4.java Mon Mar 22 07:54:51 2010
@@ -24,21 +24,14 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -51,22 +44,16 @@ import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -89,119 +76,20 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs4 extends Configured implements Tool {
-
+public class TestMultipleOutputs4 extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- //protected static ExecType execType = ExecType.MAPREDUCE;
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ public static void setUpOnce() throws Exception {
+ init();
+ inputPath = getTableFullPath(inputFileName).toString();
+ writeToFile(inputPath);
}
public String getCurrentMethodName() {
@@ -228,48 +116,8 @@ public class TestMultipleOutputs4 extend
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
+ if ( mode == TestMode.local ){
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -281,8 +129,7 @@ public class TestMultipleOutputs4 extend
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
+ } else {
FSDataOutputStream fout = fs.create(new Path (inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -307,7 +154,7 @@ public class TestMultipleOutputs4 extend
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.local ) {
System.out.println("in mini, token: "+token);
//in mini, token: file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
if (count == 1)
@@ -316,8 +163,7 @@ public class TestMultipleOutputs4 extend
strTable2 = token;
if (count == 3)
strTable3 = token;
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
System.out.println("in real, token: "+token);
//in real, token: /user/hadoopqa/ustest3
//note: no prefix file: in real cluster
@@ -331,24 +177,7 @@ public class TestMultipleOutputs4 extend
}
}
- public static void checkTableExists(boolean expected, String strDir) throws IOException{
-
- File theDir = null;
- boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
-
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path (theDir.toString()));
- }
- System.out.println("the dir : "+ theDir.toString());
- if (actual != expected){
- Assert.fail("dir exists or not is different from what expected.");
- }
- }
+
public static void checkTable(String myMultiLocs) throws IOException {
System.out.println("myMultiLocs:" + myMultiLocs);
System.out.println("sorgetTablePathst key:" + sortKey);
@@ -513,13 +342,9 @@ public class TestMultipleOutputs4 extend
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a");
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a");
- }
+
+ myMultiLocs = getTableFullPath( "a" ).toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(myMultiLocs, sortKey);
@@ -692,13 +517,10 @@ public class TestMultipleOutputs4 extend
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4TypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4TypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4TypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputs4TypedApi.java Mon Mar 22 07:54:51 2010
@@ -27,19 +27,13 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -52,23 +46,17 @@ import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,119 +79,20 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs4TypedApi extends Configured implements Tool {
-
+public class TestMultipleOutputs4TypedApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ public static void setUpOnce() throws Exception {
+ init();
+ inputPath = getTableFullPath(inputFileName).toString();
+ writeToFile(inputPath);
}
public String getCurrentMethodName() {
@@ -230,49 +119,8 @@ public class TestMultipleOutputs4TypedAp
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
- public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ public static void writeToFile(String inputFile) throws IOException {
+ if ( mode == TestMode.local ) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -284,8 +132,7 @@ public class TestMultipleOutputs4TypedAp
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -310,7 +157,7 @@ public class TestMultipleOutputs4TypedAp
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.local ) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -320,8 +167,7 @@ public class TestMultipleOutputs4TypedAp
strTable2 = token;
if (count == 3)
strTable3 = token;
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -336,27 +182,6 @@ public class TestMultipleOutputs4TypedAp
}
}
- public static void checkTableExists(boolean expected, String strDir)
- throws IOException {
-
- File theDir = null;
- boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
-
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
- System.out.println("the dir : " + theDir.toString());
-
- if (actual != expected) {
- Assert.fail("dir exists or not is different from what expected.");
- }
- }
-
public static void checkTable(String myMultiLocs) throws IOException {
System.out.println("myMultiLocs:" + myMultiLocs);
System.out.println("sorgetTablePathst key:" + sortKey);
@@ -522,17 +347,11 @@ public class TestMultipleOutputs4TypedAp
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a");
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a")));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a");
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a")));
- }
+
+ Path pathA = getTableFullPath( "a" );
+ paths.add( pathA );
+ myMultiLocs = pathA.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(sortKey, paths.toArray(new Path[1]));
@@ -708,13 +527,10 @@ public class TestMultipleOutputs4TypedAp
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4TypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypeApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypeApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypeApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypeApi.java Mon Mar 22 07:54:51 2010
@@ -20,26 +20,19 @@ package org.apache.hadoop.zebra.mapred;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -55,20 +48,15 @@ import org.apache.hadoop.mapred.TextOutp
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -92,127 +80,25 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputsTypeApi extends Configured implements Tool {
-
+public class TestMultipleOutputsTypeApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- //protected static ExecType execType = ExecType.MAPREDUCE;
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ public static void setUpOnce() throws Exception {
+ init();
+ inputPath = getTableFullPath(inputFileName).toString();
+ writeToFile(inputPath);
}
@AfterClass
public static void tearDown() throws Exception {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
pigServer.shutdown();
- }
}
public String getCurrentMethodName() {
@@ -240,7 +126,7 @@ public class TestMultipleOutputsTypeApi
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.local ) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -252,8 +138,7 @@ public class TestMultipleOutputsTypeApi
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -267,47 +152,6 @@ public class TestMultipleOutputsTypeApi
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -521,32 +365,17 @@ public class TestMultipleOutputsTypeApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
-
- }
+
+ Path pathUs = getTableFullPath( "us" + methodName );
+ Path pathIndia = getTableFullPath( "india" + methodName );
+ Path pathJapan = getTableFullPath( "japan" + methodName );
+ myMultiLocs = pathUs.toString() + "," +
+ pathIndia.toString() + "," +
+ pathJapan.toString();
+ paths.add(pathUs);
+ paths.add(pathIndia);
+ paths.add(pathJapan);
+
getTablePaths(myMultiLocs);
System.out.println("strTable1: " + strTable1.toString());
System.out.println("strTable2: " + strTable2.toString());
@@ -571,39 +400,24 @@ public class TestMultipleOutputsTypeApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- System.out.println("which cluster: " + whichCluster);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
- }
+
+ Path pathUs = getTableFullPath( "us" + methodName );
+ Path pathIndia = getTableFullPath( "india" + methodName );
+ Path pathJapan = getTableFullPath( "japan" + methodName );
+ myMultiLocs = pathUs.toString() + "," +
+ pathIndia.toString() + "," +
+ pathJapan.toString();
+ paths.add(pathUs);
+ paths.add(pathIndia);
+ paths.add(pathJapan);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
checkTable(myMultiLocs);
+ System.out.println("DONE test 2");
}
@Test
@@ -615,44 +429,27 @@ public class TestMultipleOutputsTypeApi
// setUpOnce();
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "count";
- System.out.println("hello sort on word and count, which cluster: "
- + whichCluster);
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
- }
+ Path pathUs = getTableFullPath( "us" + methodName );
+ Path pathIndia = getTableFullPath( "india" + methodName );
+ Path pathJapan = getTableFullPath( "japan" + methodName );
+ myMultiLocs = pathUs.toString() + "," +
+ pathIndia.toString() + "," +
+ pathJapan.toString();
+ paths.add(pathUs);
+ paths.add(pathIndia);
+ paths.add(pathJapan);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
checkTable(myMultiLocs);
+ System.out.println("DONE test 3");
}
static class MapClass implements
@@ -835,18 +632,15 @@ public class TestMultipleOutputsTypeApi
test.test1();
test.test2();
test.test3();
-
+
return 0;
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputsTypeApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypedApiNeg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypedApiNeg.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypedApiNeg.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestMultipleOutputsTypedApiNeg.java Mon Mar 22 07:54:51 2010
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapred.TextOutp
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
@@ -91,189 +92,26 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputsTypedApiNeg extends Configured implements Tool {
-
+public class TestMultipleOutputsTypedApiNeg extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- //protected static ExecType execType = ExecType.MAPREDUCE;
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
+ public static void setUpOnce() throws Exception {
+ init();
// set inputPath and output path
String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
+ inputPath = getTableFullPath(inputFileName).toString();
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
- }
-
- public String getCurrentMethodName() {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintWriter pw = new PrintWriter(baos);
- (new Throwable()).printStackTrace(pw);
- pw.flush();
- String stackTrace = baos.toString();
- pw.close();
-
- StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
- tok.nextToken(); // 'java.lang.Throwable'
- tok.nextToken(); // 'at ...getCurrentMethodName'
- String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
- // Parse line 3
- tok = new StringTokenizer(l.trim(), " <(");
- String t = tok.nextToken(); // 'at'
- t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
- StringTokenizer st = new StringTokenizer(t, ".");
- String methodName = null;
- while (st.hasMoreTokens()) {
- methodName = st.nextToken();
- }
- return methodName;
- }
-
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.local ) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -285,8 +123,7 @@ public class TestMultipleOutputsTypedApi
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -300,6 +137,30 @@ public class TestMultipleOutputsTypedApi
}
}
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ StringTokenizer st = new StringTokenizer(t, ".");
+ String methodName = null;
+ while (st.hasMoreTokens()) {
+ methodName = st.nextToken();
+ }
+ return methodName;
+ }
+
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -311,7 +172,7 @@ public class TestMultipleOutputsTypedApi
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.local ) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -321,8 +182,7 @@ public class TestMultipleOutputsTypedApi
strTable2 = token;
if (count == 3)
strTable3 = token;
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -337,27 +197,6 @@ public class TestMultipleOutputsTypedApi
}
}
- public static void checkTableExists(boolean expected, String strDir)
- throws IOException {
-
- File theDir = null;
- boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
-
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
- System.out.println("the dir : " + theDir.toString());
-
- if (actual != expected) {
- Assert.fail("dir exists or not is different from what expected.");
- }
- }
-
public static void checkTable(String myMultiLocs) throws IOException {
System.out.println("myMultiLocs:" + myMultiLocs);
System.out.println("sorgetTablePathst key:" + sortKey);
@@ -525,44 +364,33 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + ""
- + "," + "/user/" + System.getenv("USER") + "/" + "b" + methodName);
-
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
-
+ myMultiLocs = getTableFullPath( "a" + methodName ).toString() + "," +
+ getTableFullPath( "b" + methodName ).toString();
+
+
+ paths.add(getTableFullPath( "a" + methodName ));
+
+ if (mode == TestMode.cluster) {
try {
paths.add(new Path(""));
} catch (IllegalArgumentException e) {
System.out.println(e.getMessage());
- System.exit(0);
+ return;
}
-
- // should not reach here
- Assert.fail("Should have seen exception already");
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "b" + methodName)));
} else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "" + ","
- + fs.getWorkingDirectory() + "/" + "b" + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
paths.add(new Path(""));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "b"
- + methodName)));
}
+
+ // should not reach here
+ Assert.fail("Should have seen exception already");
+
+ paths.add(getTableFullPath( "b" + methodName ));
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
+ System.out.println("DONE test 1");
}
@Test(expected = IOException.class)
@@ -578,26 +406,15 @@ public class TestMultipleOutputsTypedApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
+
+ Path path = getTableFullPath( "a" + methodName );
+ myMultiLocs = path.toString();
+ paths.add( path );
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
-
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(sortKey, paths.toArray(new Path[1]));
+ System.out.println("DONE test 2");
}
@Test(expected = NullPointerException.class)
@@ -613,22 +430,24 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName);
- paths.add(null);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName);
- paths.add(null);
-
- }
+ myMultiLocs = getTableFullPath( "a" + methodName ).toString();
+ paths.add(null);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
- runMR(sortKey, paths.toArray(new Path[1]));
+
+ if (mode == TestMode.cluster) {
+ try {
+ runMR(sortKey, paths.toArray(new Path[1]));
+ } catch (NullPointerException e) {
+ System.err.println(e.getMessage());
+ System.out.println("DONE test 3");
+ return;
+ }
+ } else {
+ runMR(sortKey, paths.toArray(new Path[1]));
+ System.out.println("DONE test 3");
+ }
}
@Test(expected = IOException.class)
@@ -646,38 +465,21 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "b"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "b" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "b"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "b"
- + methodName)));
- }
+ Path pathA = getTableFullPath( "a" + methodName );
+ Path pathB = getTableFullPath( "b" + methodName );
+
+ myMultiLocs = pathA.toString() + "," + pathA.toString() + ","
+ + pathB.toString();
+
+ paths.add( pathA );
+ paths.add( pathA );
+ paths.add( pathB );
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
-
+ System.out.println("DONE test 4");
}
static class MapClass implements
@@ -850,21 +652,24 @@ public class TestMultipleOutputsTypedApi
TestMultipleOutputsTypedApiNeg.setUpOnce();
test.test1();
- test.test2();
+
+ //TODO: backend exception - will migrate to real cluster later
+ //test.test2();
+
test.test3();
- test.test4();
+
+ //TODO: backend exception
+ //test.test4();
return 0;
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputsTypedApiNeg(), args);
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestSmokeMR.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestSmokeMR.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestSmokeMR.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestSmokeMR.java Mon Mar 22 07:54:51 2010
@@ -22,19 +22,12 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -48,20 +41,14 @@ import org.apache.hadoop.mapred.Reporter
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -82,21 +69,12 @@ import org.junit.BeforeClass;
* </pre>
*
*/
-public class TestSmokeMR extends Configured implements Tool{
+public class TestSmokeMR extends BaseTestCase implements Tool{
static String inputPath;
static String outputPath;
static String inputFileName = "smoke.txt";
static String outputTableName ="smokeTable";
- protected static ExecType execType = ExecType.MAPREDUCE;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- private static Configuration conf;
public static String sortKey = null;
-
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
static class MapClass implements
Mapper<LongWritable, Text, BytesWritable, Tuple> {
@@ -194,98 +172,13 @@ public class TestSmokeMR extends Configu
@BeforeClass
public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- if (System.getProperty("whichCluster") == null) {
- System.setProperty("whichCluster", "realCluster");
- System.out.println("should be called");
- whichCluster = System.getProperty("whichCluster");
- } else {
- whichCluster = System.getProperty("whichCluster");
- }
-
- System.out.println("clusterddddd: " + whichCluster);
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
- if ((whichCluster.equalsIgnoreCase("realCluster") && System
- .getenv("HADOOP_HOME") == null)) {
- System.out.println("Please set HADOOP_HOME");
- System.exit(0);
- }
-
- if ( conf == null ) {
- conf = new Configuration();
- }
-
- if ((whichCluster.equalsIgnoreCase("realCluster") && System.getenv("USER") == null)) {
- System.out.println("Please set USER");
- System.exit(0);
- }
-// zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
- zebraJar = System.getenv("ZEBRA_JAR");
- File file = new File(zebraJar);
- if (!file.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put zebra.jar at hadoop_home/lib");
- // System.exit(0);
- }
-
- // set inputPath and outPath
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- outputPath = new String("/user/" + System.getenv("USER") + "/" +outputTableName);
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- outputPath = new String(workingDir + "/" + outputTableName);
- System.out.println("inputPath: " + inputPath);
-
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ inputPath = getTableFullPath( inputFileName ).toString();
+ outputPath = getTableFullPath(outputTableName).toString();
+ writeToFile(inputPath);
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if ( mode == TestMode.cluster ) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -297,8 +190,7 @@ public class TestSmokeMR extends Configu
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -311,29 +203,10 @@ public class TestSmokeMR extends Configu
fout.close();
}
}
- public static void checkTableExists(boolean expected, String strDir)
- throws IOException {
-
- File theDir = null;
- boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
-
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
- System.out.println("the dir : " + theDir.toString());
- if (actual != expected) {
- Assert.fail("dir exists or not is different from what expected.");
- }
- }
public static void removeDir(Path outPath) throws IOException {
String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ if ( mode == TestMode.cluster ) {
command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
+ outPath.toString();
} else {