You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/03/22 08:54:53 UTC
svn commit: r925988 [4/8] - in /hadoop/pig/trunk/contrib/zebra: ./
src/test/org/apache/hadoop/zebra/ src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/mapreduce/
src/test/org/apache/hadoop/zebra/pig/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2TypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2TypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2TypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2TypedApi.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -58,14 +54,12 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -86,118 +80,21 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs2TypedApi extends Configured implements Tool {
+public class TestMultipleOutputs2TypedApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -225,7 +122,7 @@ public class TestMultipleOutputs2TypedAp
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -238,7 +135,8 @@ public class TestMultipleOutputs2TypedAp
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -252,47 +150,6 @@ public class TestMultipleOutputs2TypedAp
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -304,7 +161,7 @@ public class TestMultipleOutputs2TypedAp
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -312,10 +169,9 @@ public class TestMultipleOutputs2TypedAp
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -323,8 +179,6 @@ public class TestMultipleOutputs2TypedAp
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
}
@@ -332,18 +186,12 @@ public class TestMultipleOutputs2TypedAp
public static void checkTableExists(boolean expected, String strDir)
throws IOException {
-
File theDir = null;
boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
+ theDir = new File(strDir);
+ actual = fs.exists(new Path(theDir.toString()));
+
System.out.println("the dir : " + theDir.toString());
if (actual != expected) {
@@ -515,28 +363,14 @@ public class TestMultipleOutputs2TypedAp
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- List<Path> paths = new ArrayList<Path>(2);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
-
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+ List<Path> paths = new ArrayList<Path>(2);
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
System.out.println("strTable1: " + strTable1);
removeDir(new Path(strTable1));
@@ -544,7 +378,6 @@ public class TestMultipleOutputs2TypedAp
runMR(sortKey, paths.toArray(new Path[2]));
checkTable(myMultiLocs);
System.out.println("DONE test" + getCurrentMethodName());
-
}
@Test
@@ -560,33 +393,18 @@ public class TestMultipleOutputs2TypedAp
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(2);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
-
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
runMR(sortKey, paths.toArray(new Path[2]));
checkTable(myMultiLocs);
System.out.println("DONE test" + getCurrentMethodName());
-
}
@Test
@@ -601,25 +419,13 @@ public class TestMultipleOutputs2TypedAp
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(2);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -629,7 +435,6 @@ public class TestMultipleOutputs2TypedAp
checkTableExists(true, strTable2);
checkTable(myMultiLocs);
System.out.println("DONE test" + getCurrentMethodName());
-
}
static class MapClass extends
@@ -807,13 +612,10 @@ public class TestMultipleOutputs2TypedAp
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs2TypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3.java Mon Mar 22 07:54:51 2010
@@ -30,12 +30,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -53,13 +49,11 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -80,119 +74,22 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs3 extends Configured implements Tool {
+public class TestMultipleOutputs3 extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+ inputPath = getTableFullPath(inputFileName).toString();
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -218,73 +115,34 @@ public class TestMultipleOutputs3 extend
}
return methodName;
}
- public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- FileWriter fstream = new FileWriter(inputFile);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write("us 2\n");
- out.write("japan 2\n");
- out.write("india 4\n");
- out.write("us 2\n");
- out.write("japan 1\n");
- out.write("india 3\n");
- out.write("nouse 5\n");
- out.write("nowhere 4\n");
- out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- FSDataOutputStream fout = fs.create(new Path (inputFile));
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 2\n");
- fout.writeBytes("india 4\n");
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 1\n");
- fout.writeBytes("india 3\n");
- fout.writeBytes("nouse 5\n");
- fout.writeBytes("nowhere 4\n");
- fout.close();
- }
- }
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
+ public static void writeToFile (String inputFile) throws IOException{
+ if (mode == TestMode.local) {
+ FileWriter fstream = new FileWriter(inputFile);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write("us 2\n");
+ out.write("japan 2\n");
+ out.write("india 4\n");
+ out.write("us 2\n");
+ out.write("japan 1\n");
+ out.write("india 3\n");
+ out.write("nouse 5\n");
+ out.write("nowhere 4\n");
+ out.close();
}
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
+
+ if (mode == TestMode.cluster) {
+ FSDataOutputStream fout = fs.create(new Path (inputFile));
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 2\n");
+ fout.writeBytes("india 4\n");
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 1\n");
+ fout.writeBytes("india 3\n");
+ fout.writeBytes("nouse 5\n");
+ fout.writeBytes("nowhere 4\n");
+ fout.close();
}
-
}
public static void getTablePaths(String myMultiLocs) {
@@ -302,8 +160,7 @@ public class TestMultipleOutputs3 extend
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
+
System.out.println("token = " + token);
}
}
@@ -469,22 +326,16 @@ public class TestMultipleOutputs3 extend
/*
* test combine sort keys
*/
- System.out.println("******Starttt testcase: " + getCurrentMethodName());
+ System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- }
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("others" + methodName).toString();
+ myMultiLocs = str1 + "," + str2;
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -650,13 +501,10 @@ public class TestMultipleOutputs3 extend
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs3(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3TypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3TypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3TypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs3TypedApi.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -58,14 +54,12 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -86,119 +80,22 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs3TypedApi extends Configured implements Tool{
+public class TestMultipleOutputs3TypedApi extends BaseTestCase implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+ inputPath = getTableFullPath(inputFileName).toString();
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -224,73 +121,34 @@ public class TestMultipleOutputs3TypedAp
}
return methodName;
}
- public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- FileWriter fstream = new FileWriter(inputFile);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write("us 2\n");
- out.write("japan 2\n");
- out.write("india 4\n");
- out.write("us 2\n");
- out.write("japan 1\n");
- out.write("india 3\n");
- out.write("nouse 5\n");
- out.write("nowhere 4\n");
- out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- FSDataOutputStream fout = fs.create(new Path (inputFile));
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 2\n");
- fout.writeBytes("india 4\n");
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 1\n");
- fout.writeBytes("india 3\n");
- fout.writeBytes("nouse 5\n");
- fout.writeBytes("nowhere 4\n");
- fout.close();
- }
- }
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
+ public static void writeToFile (String inputFile) throws IOException{
+ if (mode == TestMode.local) {
+ FileWriter fstream = new FileWriter(inputFile);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write("us 2\n");
+ out.write("japan 2\n");
+ out.write("india 4\n");
+ out.write("us 2\n");
+ out.write("japan 1\n");
+ out.write("india 3\n");
+ out.write("nouse 5\n");
+ out.write("nowhere 4\n");
+ out.close();
}
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
+
+ if (mode == TestMode.cluster) {
+ FSDataOutputStream fout = fs.create(new Path (inputFile));
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 2\n");
+ fout.writeBytes("india 4\n");
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 1\n");
+ fout.writeBytes("india 3\n");
+ fout.writeBytes("nouse 5\n");
+ fout.writeBytes("nowhere 4\n");
+ fout.close();
}
-
}
public static void getTablePaths(String myMultiLocs) {
@@ -308,8 +166,6 @@ public class TestMultipleOutputs3TypedAp
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
System.out.println("token = " + token);
}
}
@@ -482,22 +338,12 @@ public class TestMultipleOutputs3TypedAp
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(2);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/" + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/" + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
-
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us" + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others" + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -668,13 +514,10 @@ public class TestMultipleOutputs3TypedAp
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs3TypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4.java Mon Mar 22 07:54:51 2010
@@ -30,12 +30,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -53,13 +49,11 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -80,120 +74,22 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs4 extends Configured implements Tool{
+public class TestMultipleOutputs4 extends BaseTestCase implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -220,71 +116,32 @@ public class TestMultipleOutputs4 extend
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- FileWriter fstream = new FileWriter(inputFile);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write("us 2\n");
- out.write("japan 2\n");
- out.write("india 4\n");
- out.write("us 2\n");
- out.write("japan 1\n");
- out.write("india 3\n");
- out.write("nouse 5\n");
- out.write("nowhere 4\n");
- out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- FSDataOutputStream fout = fs.create(new Path (inputFile));
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 2\n");
- fout.writeBytes("india 4\n");
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 1\n");
- fout.writeBytes("india 3\n");
- fout.writeBytes("nouse 5\n");
- fout.writeBytes("nowhere 4\n");
- fout.close();
+ if (mode == TestMode.local) {
+ FileWriter fstream = new FileWriter(inputFile);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write("us 2\n");
+ out.write("japan 2\n");
+ out.write("india 4\n");
+ out.write("us 2\n");
+ out.write("japan 1\n");
+ out.write("india 3\n");
+ out.write("nouse 5\n");
+ out.write("nowhere 4\n");
+ out.close();
+ }
+
+ if (mode == TestMode.cluster) {
+ FSDataOutputStream fout = fs.create(new Path (inputFile));
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 2\n");
+ fout.writeBytes("india 4\n");
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 1\n");
+ fout.writeBytes("india 3\n");
+ fout.writeBytes("nouse 5\n");
+ fout.writeBytes("nowhere 4\n");
+ fout.close();
}
}
@@ -299,17 +156,16 @@ public class TestMultipleOutputs4 extend
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: "+token);
//in mini, token: file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
if (count == 1)
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: "+token);
//in real, token: /user/hadoopqa/ustest3
//note: no prefix file: in real cluster
@@ -317,30 +173,24 @@ public class TestMultipleOutputs4 extend
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
}
}
- public static void checkTableExists(boolean expected, String strDir) throws IOException{
- File theDir = null;
- boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
+ public static void checkTableExists(boolean expected, String strDir) throws IOException{
+ File theDir = null;
+ boolean actual = false;
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path (theDir.toString()));
- }
- System.out.println("the dir : "+ theDir.toString());
- if (actual != expected){
- Assert.fail("dir exists or not is different from what expected.");
- }
- }
+ theDir = new File(strDir);
+ actual = fs.exists(new Path (theDir.toString()));
+
+ System.out.println("the dir : "+ theDir.toString());
+ if (actual != expected){
+ Assert.fail("dir exists or not is different from what expected.");
+ }
+ }
+
public static void checkTable(String myMultiLocs) throws IOException {
System.out.println("myMultiLocs:" + myMultiLocs);
System.out.println("sorgetTablePathst key:" + sortKey);
@@ -503,21 +353,15 @@ public class TestMultipleOutputs4 extend
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "word,count";
System.out.println("hello sort on word and count");
- String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a");
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a");
- }
+
+ myMultiLocs = getTableFullPath("a").toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(myMultiLocs, sortKey);
checkTableExists(true, strTable1);
System.out.println("DONE test " + getCurrentMethodName());
-
}
@@ -671,13 +515,10 @@ public class TestMultipleOutputs4 extend
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4TypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4TypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4TypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs4TypedApi.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -58,14 +54,12 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -86,120 +80,23 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs4TypedApi extends Configured implements Tool {
+public class TestMultipleOutputs4TypedApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
- public static String sortKey = null;
- private static FileSystem fs;
+ public static String sortKey = null;
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+ inputPath = getTableFullPath(inputFileName).toString();
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -226,49 +123,8 @@ public class TestMultipleOutputs4TypedAp
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -281,7 +137,8 @@ public class TestMultipleOutputs4TypedAp
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -306,7 +163,7 @@ public class TestMultipleOutputs4TypedAp
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -314,10 +171,9 @@ public class TestMultipleOutputs4TypedAp
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -325,10 +181,7 @@ public class TestMultipleOutputs4TypedAp
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
-
}
}
@@ -337,15 +190,10 @@ public class TestMultipleOutputs4TypedAp
File theDir = null;
boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
+ theDir = new File(strDir);
+ actual = fs.exists(new Path(theDir.toString()));
+
System.out.println("the dir : " + theDir.toString());
if (actual != expected) {
@@ -515,26 +363,18 @@ public class TestMultipleOutputs4TypedAp
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "word,count";
System.out.println("hello sort on word and count");
- String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
+
+ Path path1 = getTableFullPath("a");
+ paths.add(path1);
+ myMultiLocs = path1.toString();
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a");
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a")));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a");
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a")));
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(sortKey, paths.toArray(new Path[1]));
checkTableExists(true, strTable1);
System.out.println("DONE test " + getCurrentMethodName());
-
}
static class MapClass extends
@@ -692,13 +532,10 @@ public class TestMultipleOutputs4TypedAp
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4TypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypeApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypeApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypeApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypeApi.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -48,6 +44,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
import org.apache.hadoop.zebra.mapreduce.ZebraSchema;
@@ -58,12 +55,9 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -87,125 +81,28 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputsTypeApi extends Configured implements Tool{
+public class TestMultipleOutputsTypeApi extends BaseTestCase implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
@AfterClass
public static void tearDown() throws Exception {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
pigServer.shutdown();
}
}
@@ -235,7 +132,7 @@ public class TestMultipleOutputsTypeApi
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -248,7 +145,8 @@ public class TestMultipleOutputsTypeApi
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -262,47 +160,6 @@ public class TestMultipleOutputsTypeApi
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -516,35 +373,19 @@ public class TestMultipleOutputsTypeApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("india" + methodName);
+ Path path3 = getTableFullPath("japan" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString() + "," + path3.toString();
+ paths.add(path1);
+ paths.add(path2);
+ paths.add(path3);
+
getTablePaths(myMultiLocs);
System.out.println("strTable1: " + strTable1.toString());
System.out.println("strTable2: " + strTable2.toString());
+ System.out.println("strTable3: " + strTable3.toString());
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
@@ -566,39 +407,22 @@ public class TestMultipleOutputsTypeApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- System.out.println("which cluster: " + whichCluster);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("india" + methodName);
+ Path path3 = getTableFullPath("japan" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString() + "," + path3.toString();
+ paths.add(path1);
+ paths.add(path2);
+ paths.add(path3);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
checkTable(myMultiLocs);
+ System.out.println("DONE test 2");
}
@Test
@@ -607,47 +431,27 @@ public class TestMultipleOutputsTypeApi
/*
* test 'count' sort key
*/
- // setUpOnce();
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "count";
- System.out.println("hello sort on word and count, which cluster: "
- + whichCluster);
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "india" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "japan" + methodName)));
- } else {
-
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "india"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "japan"
- + methodName)));
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("india" + methodName);
+ Path path3 = getTableFullPath("japan" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString() + "," + path3.toString();
+ paths.add(path1);
+ paths.add(path2);
+ paths.add(path3);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
checkTable(myMultiLocs);
+ System.out.println("DONE test 3");
}
static class MapClass extends
@@ -821,13 +625,10 @@ public class TestMultipleOutputsTypeApi
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputsTypeApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}