You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/03/22 08:54:53 UTC
svn commit: r925988 [5/8] - in /hadoop/pig/trunk/contrib/zebra: ./
src/test/org/apache/hadoop/zebra/ src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/mapreduce/
src/test/org/apache/hadoop/zebra/pig/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypedApiNeg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypedApiNeg.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypedApiNeg.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputsTypedApiNeg.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -48,6 +44,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
import org.apache.hadoop.zebra.mapreduce.ZebraSchema;
@@ -58,12 +55,9 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -86,119 +80,23 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputsTypedApiNeg extends Configured implements Tool{
+public class TestMultipleOutputsTypedApiNeg extends BaseTestCase implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
-
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
+
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
+
+ inputPath = getTableFullPath(inputFileName).toString();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -225,49 +123,8 @@ public class TestMultipleOutputsTypedApi
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -280,7 +137,8 @@ public class TestMultipleOutputsTypedApi
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -305,7 +163,7 @@ public class TestMultipleOutputsTypedApi
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -316,7 +174,8 @@ public class TestMultipleOutputsTypedApi
if (count == 3)
strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -336,15 +195,10 @@ public class TestMultipleOutputsTypedApi
File theDir = null;
boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
+ theDir = new File(strDir);
+ actual = fs.exists(new Path(theDir.toString()));
+
System.out.println("the dir : " + theDir.toString());
if (actual != expected) {
@@ -518,43 +372,35 @@ public class TestMultipleOutputsTypedApi
String methodName = getCurrentMethodName();
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + ""
- + "," + "/user/" + System.getenv("USER") + "/" + "b" + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
-
+
+
+ Path path1 = getTableFullPath("a" + methodName);
+ paths.add(path1);
+
+ if (mode == TestMode.cluster) {
try {
paths.add(new Path(""));
} catch (IllegalArgumentException e) {
System.out.println(e.getMessage());
+ System.out.println("DONE test 1");
return;
}
-
- // should not reach here
- Assert.fail("Should have seen exception already");
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "b" + methodName)));
} else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "" + ","
- + fs.getWorkingDirectory() + "/" + "b" + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
paths.add(new Path(""));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "b"
- + methodName)));
}
+
+ // should not reach here
+ Assert.fail("Should have seen exception already");
+
+ Path path3 = getTableFullPath("b" + methodName);
+ paths.add(path3);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(sortKey, paths.toArray(new Path[3]));
+ System.out.println("DONE test 1");
}
// TODO: No exception is thrown any more due to Hadoop 20 API. This test case
@@ -573,22 +419,10 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
-
- }
+ Path path1 = getTableFullPath("a" + methodName);
+ paths.add(path1);
+ myMultiLocs = path1.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
runMR(sortKey, paths.toArray(new Path[1]));
@@ -607,27 +441,19 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(1);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName);
- paths.add(null);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName);
- paths.add(null);
-
- }
+ Path path1 = getTableFullPath("a" + methodName);
+ paths.add(null);
+ myMultiLocs = path1.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, paths.toArray(new Path[1]));
} catch (NullPointerException e) {
System.err.println(e.getMessage());
+ System.out.println("DONE test 3");
return;
}
} else {
@@ -650,38 +476,20 @@ public class TestMultipleOutputsTypedApi
String myMultiLocs = null;
List<Path> paths = new ArrayList<Path>(3);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "b"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "b" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "b"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "b"
- + methodName)));
- }
+ Path path1 = getTableFullPath("a" + methodName);
+ Path path2 = getTableFullPath("a" + methodName);
+ Path path3 = getTableFullPath("b" + methodName);
+ paths.add(path1);
+ paths.add(path2);
+ paths.add(path3);
+ myMultiLocs = path1.toString() + "," +path2.toString() + "," + path3.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, paths.toArray(new Path[3]));
} catch (IOException e) {
@@ -864,13 +672,10 @@ public class TestMultipleOutputsTypedApi
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputsTypedApiNeg(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTempDirRemoval.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTempDirRemoval.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTempDirRemoval.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTempDirRemoval.java Mon Mar 22 07:54:51 2010
@@ -34,9 +34,7 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -45,6 +43,9 @@ import org.apache.hadoop.mapreduce.Mappe
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
import org.apache.hadoop.zebra.mapreduce.ZebraSchema;
@@ -55,12 +56,9 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -83,117 +81,21 @@ import org.junit.Test;
*
*
*/
-public class TestTempDirRemoval {
-
+public class TestTempDirRemoval extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- if (System.getProperty("whichCluster") == null) {
- System.setProperty("whichCluster", "miniCluster");
- System.out.println("should be called");
- whichCluster = System.getProperty("whichCluster");
- } else {
- whichCluster = System.getProperty("whichCluster");
- }
-
- System.out.println("clusterddddd: " + whichCluster);
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
- if ((whichCluster.equalsIgnoreCase("realCluster") && System
- .getenv("HADOOP_HOME") == null)) {
- System.out.println("Please set HADOOP_HOME");
- System.exit(0);
- }
-
- conf = new Configuration();
+ public static void setUpOnce() throws Exception {
+ init();
+
+ inputPath = getTableFullPath(inputFileName).toString();
- if ((whichCluster.equalsIgnoreCase("realCluster") && System.getenv("USER") == null)) {
- System.out.println("Please set USER");
- System.exit(0);
- }
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put zebra.jar at hadoop_home/lib");
- System.exit(0);
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -220,49 +122,8 @@ public class TestTempDirRemoval {
return methodName;
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -275,7 +136,8 @@ public class TestTempDirRemoval {
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -300,7 +162,7 @@ public class TestTempDirRemoval {
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: " + token);
// in mini, token:
// file:/homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
@@ -308,10 +170,9 @@ public class TestTempDirRemoval {
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: " + token);
// in real, token: /user/hadoopqa/ustest3
// note: no prefix file: in real cluster
@@ -319,8 +180,7 @@ public class TestTempDirRemoval {
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
+
}
}
@@ -331,15 +191,10 @@ public class TestTempDirRemoval {
File theDir = null;
boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path(theDir.toString()));
- }
+ theDir = new File(strDir);
+ actual = fs.exists(new Path(theDir.toString()));
+
System.out.println("the dir : " + theDir.toString());
if (actual != expected) {
@@ -514,33 +369,19 @@ public class TestTempDirRemoval {
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- List<Path> paths = new ArrayList<Path>(3);
+ List<Path> paths = new ArrayList<Path>(2);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "a"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "b"
- + methodName);
-
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "a" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "b" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "b"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "a"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "b"
- + methodName)));
- }
+ Path path1 = getTableFullPath("a" + methodName);
+ Path path2 = getTableFullPath("b" + methodName);
+ paths.add(path1);
+ paths.add(path2);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
runMR(sortKey, paths.toArray(new Path[2]));
-
+ System.out.println("DONE test 4");
}
static class MapClass extends
@@ -660,8 +501,9 @@ public class TestTempDirRemoval {
public void runMR(String sortKey, Path... paths) throws ParseException,
IOException, Exception, org.apache.hadoop.zebra.parser.ParseException {
- Job job = new Job();
+ Job job = new Job(conf);
job.setJobName("tableMRSample");
+ job.setJarByClass(TestTempDirRemoval.class);
Configuration conf = job.getConfiguration();
conf.set("table.output.tfile.compression", "gz");
conf.set("sortKey", sortKey);
@@ -707,14 +549,24 @@ public class TestTempDirRemoval {
throw new RuntimeException("Temp Dir sld not exist after BTOF.close()" + tmpPath.toString());
}
}
-
}
-
- public static void main(String[] args) throws ParseException,
- org.apache.hadoop.zebra.parser.ParseException, Exception {
+
+ @Override
+ public int run(String[] args) throws Exception {
TestTempDirRemoval test = new TestTempDirRemoval();
TestTempDirRemoval.setUpOnce();
-
+
test.test4();
+
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ conf = new Configuration();
+
+ int res = ToolRunner.run(conf, new TestTempDirRemoval(), args);
+ System.out.println("PASS");
+ System.exit(res);
}
+
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi.java Mon Mar 22 07:54:51 2010
@@ -33,12 +33,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
@@ -51,17 +47,15 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,122 +85,28 @@ import org.apache.hadoop.zebra.mapreduce
*
*
*/
-public class TestTypedApi extends Configured implements Tool {
+public class TestTypedApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
+
+ writeToFile(inputPath);
}
@AfterClass
public static void tearDownOnce() throws Exception {
pigServer.shutdown();
+
if (strTable1 != null) {
BasicTable.drop(new Path(strTable1), conf);
}
@@ -243,7 +143,7 @@ public class TestTypedApi extends Config
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -256,7 +156,8 @@ public class TestTypedApi extends Config
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -270,47 +171,6 @@ public class TestTypedApi extends Config
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -492,31 +352,19 @@ public class TestTypedApi extends Config
* test positive test case. schema, projection, sortInfo are all good ones.
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -534,49 +382,37 @@ public class TestTypedApi extends Config
* test negative test case. wrong schema fomat: schema = "{, count:int";
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "{, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 2");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 2");
+
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 2");
@@ -590,108 +426,39 @@ public class TestTypedApi extends Config
* test negative test case. non-exist sort key
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "not exist";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- }
- getTablePaths(myMultiLocs);
- removeDir(new Path(strTable1));
- removeDir(new Path(strTable2));
- String schema = "word:string, count:int";
- String storageHint = "[word];[count]";
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
- if (whichCluster.equals("realCluster")) {
- try {
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- } catch (IOException e) {
- System.out.println(e.getMessage());
- return;
- }
- // should not reach here
- Assert.fail("in try, should have thrown exception");
- System.out.println("done test 3");
- } else {
- runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- System.out.println("done test 3");
- }
- }
-
- @Test(expected = IOException.class)
- public void test4() throws ParseException, IOException,
- org.apache.hadoop.zebra.parser.ParseException, Exception {
- /*
- * test negative test case. sort key is empty string
- */
- System.out.println("******Start testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
-
- sortKey = "";
- System.out.println("hello sort on word and count");
- String methodName = getCurrentMethodName();
- String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (IOException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 3");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 4");
+
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- System.out.println("done test 4");
+ System.out.println("done test 3");
}
}
@@ -702,49 +469,36 @@ public class TestTypedApi extends Config
* test negative test case. sort key null
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = null;
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (NullPointerException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 5");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 5");
+
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 5");
@@ -758,33 +512,19 @@ public class TestTypedApi extends Config
* test negative test case. storage hint: none exist column
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -792,16 +532,16 @@ public class TestTypedApi extends Config
String storageHint = "[none-exist-column]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 6");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 6");
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 6");
@@ -816,33 +556,19 @@ public class TestTypedApi extends Config
* [
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
-
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -850,16 +576,16 @@ public class TestTypedApi extends Config
String storageHint = "none-exist-column]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 7");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 7");
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 7");
@@ -874,33 +600,19 @@ public class TestTypedApi extends Config
* user input has only two fields
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -908,17 +620,17 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 8");
return;
}
// should not reach here
- Assert.fail("in try, should have thrown exception");
- System.out.println("done test 8");
+ Assert.fail("in try, should have thrown exception");
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 8");
@@ -933,33 +645,19 @@ public class TestTypedApi extends Config
* inttt instead of int
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
-
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -967,17 +665,18 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 9");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 9");
+
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 9");
@@ -993,33 +692,19 @@ public class TestTypedApi extends Config
* instead of ,
*/
System.out.println("******Starttt testcase: " + getCurrentMethodName());
- List<Path> paths = new ArrayList<Path>(1);
+ List<Path> paths = new ArrayList<Path>(2);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
-
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+ paths.add(path1);
+ paths.add(path2);
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -1027,18 +712,17 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if (mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
System.out.println(e.getMessage());
+ System.out.println("done test 10");
return;
}
// should not reach here
Assert.fail("in try, should have thrown exception");
- System.out.println("done test 10");
-
} else {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
System.out.println("done test 10");
@@ -1219,7 +903,7 @@ public class TestTypedApi extends Config
test.test2();
test.test3();
- // backend exception - will migrate later
+ //TODO: backend exception - will migrate later
//test.test4();
test.test5();
@@ -1233,13 +917,10 @@ public class TestTypedApi extends Config
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestTypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestTypedApi2.java Mon Mar 22 07:54:51 2010
@@ -32,12 +32,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
@@ -50,6 +46,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
import org.apache.hadoop.zebra.mapreduce.ZebraSchema;
@@ -60,12 +57,9 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -88,117 +82,21 @@ import org.junit.Test;
*
*
*/
-public class TestTypedApi2 extends Configured implements Tool{
+public class TestTypedApi2 extends BaseTestCase implements Tool{
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
+
+ writeToFile(inputPath);
}
public String getCurrentMethodName() {
@@ -226,7 +124,7 @@ public class TestTypedApi2 extends Confi
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -239,7 +137,8 @@ public class TestTypedApi2 extends Confi
out.write("nowhere 4\n");
out.close();
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ if (mode == TestMode.cluster) {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -253,47 +152,6 @@ public class TestTypedApi2 extends Confi
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -309,8 +167,7 @@ public class TestTypedApi2 extends Confi
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
+
System.out.println("token = " + token);
}
}
@@ -474,37 +331,25 @@ public class TestTypedApi2 extends Confi
/*
* test positive test case. User defined comparator class
*/
- System.out.println("******Starttt testcase: " + getCurrentMethodName());
+ System.out.println("******Start testcase: " + getCurrentMethodName());
List<Path> paths = new ArrayList<Path>(3);
sortKey = "word,count";
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+
+ Path path1 = getTableFullPath("us" + methodName);
+ Path path2 = getTableFullPath("others" + methodName);
+ paths.add(path1);
+ paths.add(path2);
+ myMultiLocs = path1.toString() + "," + path2.toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- String sortInfo = null;
+
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
// runMR( sortKey, schema, storageHint, myMultiLocs);
checkTable(myMultiLocs);
@@ -690,13 +535,10 @@ public class TestTypedApi2 extends Confi
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestTypedApi2(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java Mon Mar 22 07:54:51 2010
@@ -18,67 +18,38 @@
package org.apache.hadoop.zebra.pig;
-import java.io.File;
import java.io.IOException;
import java.util.Iterator;
-import java.util.List;
-
import junit.framework.Assert;
-import junit.framework.TestCase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableInserter;
-import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit;
-import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
-import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-public class TestBasicTableUnionLoader {
- protected static ExecType execType = ExecType.MAPREDUCE;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- private static Path pathWorking, pathTable1, pathTable2;
- private static Configuration conf;
+public class TestBasicTableUnionLoader extends BaseTestCase {
+ private static Path pathTable1, pathTable2;
+
@BeforeClass
public static void setUp() throws Exception {
- if (System.getProperty("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
-
- conf = new Configuration();
- FileSystem fs = cluster.getFileSystem();
- pathWorking = fs.getWorkingDirectory();
+ init();
+
+ pathTable1 = getTableFullPath("TestBasicTableUnionLoader1");
+ pathTable2 = getTableFullPath("TestBasicTableUnionLoader2");
+ removeDir(pathTable1);
+ removeDir(pathTable2);
/*
* create 1st basic table;
*/
- pathTable1 = new Path(pathWorking, "TestBasicTableUnion" + "1");
- System.out.println("pathTable1 =" + pathTable1);
BasicTable.Writer writer = new BasicTable.Writer(pathTable1,
"a:string,b,c:string", "[a,b];[c]", conf);
@@ -109,12 +80,10 @@ public class TestBasicTableUnionLoader {
inserters[i].close();
}
writer.close();
-
+
/*
* create 2nd basic table;
*/
- pathTable2 = new Path(pathWorking, "TestBasicTableUnion" + "2");
- System.out.println("pathTable2 =" + pathTable2);
writer = new BasicTable.Writer(pathTable2, "a:string,b,d:string",
"[a,b];[d]", conf);
@@ -142,7 +111,7 @@ public class TestBasicTableUnionLoader {
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
- writer.close();
+ writer.close();
}
@AfterClass
@@ -152,14 +121,9 @@ public class TestBasicTableUnionLoader {
@Test
public void testReader() throws ExecException, IOException {
- /*
- * remove hdfs prefix part like "hdfs://localhost.localdomain:42540" pig
- * will fill that in.
- */
- String str1 = pathTable1.toString().substring(
- pathTable1.toString().indexOf("/", 7), pathTable1.toString().length());
- String str2 = pathTable2.toString().substring(
- pathTable2.toString().indexOf("/", 7), pathTable2.toString().length());
+ String str1 = pathTable1.toString();
+ String str2 = pathTable2.toString();
+
String query = "records = LOAD '" + str1 + "," + str2
+ "' USING org.apache.hadoop.zebra.pig.TableLoader('a, b, c, d');";
System.out.println(query);
@@ -170,22 +134,37 @@ public class TestBasicTableUnionLoader {
Tuple cur;
while (it.hasNext()) {
cur = it.next();
- System.out.println(cur);
cnt++;
+
if (cnt == 1) {
- Assert.assertEquals("0_00", cur.get(0));
- Assert.assertEquals("0_01", cur.get(1));
- Assert.assertEquals("0_02", cur.get(2));
- Assert.assertEquals(null, cur.get(3));
+ String col0 = (String)cur.get(0);
+ Assert.assertTrue(col0.equals("0_00") || col0.equals("0_10"));
+
+ String col1 = (String)cur.get(1);
+ Assert.assertTrue(col1.equals("0_01") || col1.equals("0_11"));
+
+ String col2 = (String)cur.get(2);
+ Assert.assertTrue(col2 == null || col2.equals("0_02") || col2.equals("0_12"));
+
+ String col3 = (String)cur.get(3);
+ Assert.assertTrue(col3 == null || col3.equals("0_02") || col3.equals("0_12"));
}
+
if (cnt == 21) {
- Assert.assertEquals("0_00", cur.get(0));
- Assert.assertEquals("0_01", cur.get(1));
- Assert.assertEquals(null, cur.get(2));
- Assert.assertEquals("0_02", cur.get(3));
+ String col0 = (String)cur.get(0);
+ Assert.assertTrue(col0.equals("0_00") || col0.equals("0_10"));
+
+ String col1 = (String)cur.get(1);
+ Assert.assertTrue(col1.equals("0_01") || col1.equals("0_11"));
+
+ String col2 = (String)cur.get(2);
+ Assert.assertTrue(col2 == null || col2.equals("0_02") || col2.equals("0_12"));
+
+ String col3 = (String)cur.get(3);
+ Assert.assertTrue(col3 == null || col3.equals("0_02") || col3.equals("0_12"));
}
}
+
Assert.assertEquals(cnt, 40);
}
-
}