You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/03/22 08:54:53 UTC
svn commit: r925988 [3/8] - in /hadoop/pig/trunk/contrib/zebra: ./
src/test/org/apache/hadoop/zebra/ src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/mapreduce/
src/test/org/apache/hadoop/zebra/pig/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi.java Mon Mar 22 07:54:51 2010
@@ -20,7 +20,6 @@ package org.apache.hadoop.zebra.mapred;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -28,25 +27,17 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
-
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.file.tfile.RawComparable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -55,31 +46,23 @@ import org.apache.hadoop.mapred.OutputCo
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.hadoop.zebra.mapred.ZebraSchema;
-import org.apache.hadoop.zebra.mapred.ZebraProjection;
import org.apache.hadoop.zebra.mapred.ZebraSortInfo;
import org.apache.hadoop.zebra.mapred.ZebraStorageHint;
@@ -102,118 +85,20 @@ import org.apache.hadoop.zebra.mapred.Ze
*
*
*/
-public class TestTypedApi extends Configured implements Tool {
-
+public class TestTypedApi extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf = null;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
+ public static void setUpOnce() throws Exception {
+ init();
+ inputPath = getTableFullPath(inputFileName).toString();
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
@AfterClass
@@ -255,7 +140,7 @@ public class TestTypedApi extends Config
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if( mode == TestMode.local) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -267,8 +152,7 @@ public class TestTypedApi extends Config
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -282,47 +166,6 @@ public class TestTypedApi extends Config
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -510,35 +353,19 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- String sortInfo = null;
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
checkTable(myMultiLocs);
System.out.println("DONE test " + getCurrentMethodName());
-
}
@Test(expected = ParseException.class)
@@ -554,34 +381,18 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "{, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if( mode == TestMode.cluster) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -610,34 +421,18 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (IOException e) {
@@ -666,26 +461,10 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- }
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
@@ -693,7 +472,7 @@ public class TestTypedApi extends Config
String schema = "word:string, count:int";
String storageHint = "[word];[count]";
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (IOException e) {
@@ -722,27 +501,11 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -750,7 +513,7 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (NullPointerException e) {
@@ -780,27 +543,11 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -808,7 +555,7 @@ public class TestTypedApi extends Config
String storageHint = "[none-exist-column]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -839,34 +586,18 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:int";
String storageHint = "none-exist-column]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -896,27 +627,11 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -924,7 +639,7 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -954,34 +669,18 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
- }
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
String schema = "word:string, count:inttt";
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -1011,26 +710,11 @@ public class TestTypedApi extends Config
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -1038,7 +722,7 @@ public class TestTypedApi extends Config
String storageHint = "[word];[count]";
//runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
- if (whichCluster.equals("realCluster")) {
+ if ( mode == TestMode.cluster ) {
try {
runMR(sortKey, schema, storageHint, paths.toArray(new Path[2]));
} catch (ParseException e) {
@@ -1260,13 +944,10 @@ public class TestTypedApi extends Config
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestTypedApi(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi2.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi2.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestTypedApi2.java Mon Mar 22 07:54:51 2010
@@ -20,7 +20,6 @@ package org.apache.hadoop.zebra.mapred;
import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -28,25 +27,18 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.StringTokenizer;
-import java.util.TreeMap;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.file.tfile.RawComparable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -55,29 +47,20 @@ import org.apache.hadoop.mapred.OutputCo
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.lib.MultipleOutputs;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
-import org.apache.hadoop.zebra.mapred.TestBasicTableIOFormatLocalFS.InvIndex;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.hadoop.zebra.mapred.ZebraSchema;
-import org.apache.hadoop.zebra.mapred.ZebraProjection;
import org.apache.hadoop.zebra.mapred.ZebraSortInfo;
import org.apache.hadoop.zebra.mapred.ZebraStorageHint;
@@ -100,118 +83,21 @@ import org.apache.hadoop.zebra.mapred.Ze
*
*
*/
-public class TestTypedApi2 extends Configured implements Tool {
+public class TestTypedApi2 extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf = null;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
-
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
- writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ public static void setUpOnce() throws Exception {
+ init();
+ inputPath = getTableFullPath(inputFileName).toString();
+ writeToFile(inputPath);
}
public String getCurrentMethodName() {
@@ -239,7 +125,7 @@ public class TestTypedApi2 extends Confi
}
public static void writeToFile(String inputFile) throws IOException {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if( BaseTestCase.mode == TestMode.local ) {
FileWriter fstream = new FileWriter(inputFile);
BufferedWriter out = new BufferedWriter(fstream);
out.write("us 2\n");
@@ -251,8 +137,7 @@ public class TestTypedApi2 extends Confi
out.write("nouse 5\n");
out.write("nowhere 4\n");
out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+ } else {
FSDataOutputStream fout = fs.create(new Path(inputFile));
fout.writeBytes("us 2\n");
fout.writeBytes("japan 2\n");
@@ -266,47 +151,6 @@ public class TestTypedApi2 extends Confi
}
}
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
-
- }
-
public static void getTablePaths(String myMultiLocs) {
StringTokenizer st = new StringTokenizer(myMultiLocs, ",");
@@ -493,25 +337,11 @@ public class TestTypedApi2 extends Confi
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/"
- + "others" + methodName);
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "us" + methodName)));
- paths.add(new Path(new String("/user/" + System.getenv("USER") + "/"
- + "others" + methodName)));
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName);
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName)));
- paths.add(new Path(new String(fs.getWorkingDirectory() + "/" + "others"
- + methodName)));
- }
+
+ paths.add( getTableFullPath( "us" + methodName ) );
+ paths.add( getTableFullPath( "others" + methodName ) );
+ myMultiLocs = paths.get(0).toString() + "," + paths.get(1).toString();
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -719,7 +549,7 @@ public class TestTypedApi2 extends Confi
conf = new Configuration();
int res = ToolRunner.run(conf, new TestTypedApi2(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/ToolTestComparator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/ToolTestComparator.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/ToolTestComparator.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/ToolTestComparator.java Mon Mar 22 07:54:51 2010
@@ -1,42 +1,30 @@
package org.apache.hadoop.zebra.mapred;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.Serializable;
import java.util.HashMap;
import java.util.Iterator;
import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.io.BasicTable;
import org.apache.hadoop.zebra.io.TableInserter;
import org.apache.hadoop.zebra.io.TableScanner;
-import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit;
import org.apache.hadoop.zebra.parser.ParseException;
import org.apache.hadoop.zebra.pig.TableStorer;
import org.apache.hadoop.zebra.schema.Schema;
import org.apache.hadoop.zebra.types.TypesUtils;
-import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.executionengine.ExecJob;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.Assert;
/**
@@ -45,7 +33,7 @@ import org.junit.Assert;
* Utility for verifying tables created during Zebra Stress Testing
*
*/
-public class ToolTestComparator {
+public class ToolTestComparator extends BaseTestCase {
final static String TABLE_SCHEMA = "count:int,seed:int,int1:int,int2:int,str1:string,str2:string,byte1:bytes,"
+ "byte2:bytes,float1:float,long1:long,double1:double,m1:map(string),r1:record(f1:string, f2:string),"
@@ -54,17 +42,7 @@ public class ToolTestComparator {
private static Random generator = new Random();
- private static Configuration conf;
- private static FileSystem fs;
-
- protected static ExecType execType = ExecType.MAPREDUCE;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
protected static ExecJob pigJob;
- private static Path path;
-
- private static String zebraJar;
- private static String whichCluster;
private static int totalNumbCols;
private static long totalNumbVerifiedRows;
@@ -73,65 +51,7 @@ public class ToolTestComparator {
* Setup and initialize environment
*/
public static void setUp() throws Exception {
- System.out.println("setUp()");
- if (System.getProperty("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
-
- if (System.getProperty("whichCluster") == null) {
- System.setProperty("whichCluster", "miniCluster");
- whichCluster = System.getProperty("whichCluster");
- } else {
- whichCluster = System.getProperty("whichCluster");
- }
-
- System.out.println("cluster: " + whichCluster);
- if (whichCluster.equalsIgnoreCase("realCluster")
- && System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME");
- System.exit(0);
- }
-
- conf = new Configuration();
-
- if (whichCluster.equalsIgnoreCase("realCluster")
- && System.getenv("USER") == null) {
- System.out.println("Please set USER");
- System.exit(0);
- }
- zebraJar = System.getenv("HADOOP_HOME") + "/../jars/zebra.jar";
- File file = new File(zebraJar);
- if (!file.exists() && whichCluster.equalsIgnoreCase("realCulster")) {
- System.out.println("Please put zebra.jar at hadoop_home/../jars");
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Running realCluster");
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
- path = new Path("/user/" + System.getenv("USER") + "/TestComparator");
- // removeDir(path);
- fs = path.getFileSystem(conf);
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out.println("Running miniCluster");
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
- path = new Path(fs.getWorkingDirectory() + "/TestComparator");
- // removeDir(path);
- System.out.println("path1 =" + path);
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
+ init();
}
/**
@@ -817,28 +737,6 @@ public class ToolTestComparator {
}
/**
- * Remove directory
- *
- */
- public static void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- command = "rm -rf " + outPath.toString();
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
- }
- }
-
- /**
* Calculate elapsed time
*
*/
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs.java Mon Mar 22 07:54:51 2010
@@ -30,9 +30,7 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
@@ -46,7 +44,7 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.zebra.mapred.TestTypedApi;
+import org.apache.hadoop.zebra.BaseTestCase;
import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat;
import org.apache.hadoop.zebra.mapreduce.ZebraOutputPartition;
import org.apache.hadoop.zebra.parser.ParseException;
@@ -54,11 +52,8 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -82,126 +77,32 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs extends Configured implements Tool {
+public class TestMultipleOutputs extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf = null;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
-
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
+ inputPath = getTableFullPath(inputFileName).toString();
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
+
@AfterClass
public static void tearDown() throws Exception {
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- pigServer.shutdown();
+ if (mode == TestMode.local) {
+ pigServer.shutdown();
}
}
+
public String getCurrentMethodName() {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
@@ -227,72 +128,32 @@ public class TestMultipleOutputs extends
}
public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- FileWriter fstream = new FileWriter(inputFile);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write("us 2\n");
- out.write("japan 2\n");
- out.write("india 4\n");
- out.write("us 2\n");
- out.write("japan 1\n");
- out.write("india 3\n");
- out.write("nouse 5\n");
- out.write("nowhere 4\n");
- out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- FSDataOutputStream fout = fs.create(new Path (inputFile));
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 2\n");
- fout.writeBytes("india 4\n");
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 1\n");
- fout.writeBytes("india 3\n");
- fout.writeBytes("nouse 5\n");
- fout.writeBytes("nowhere 4\n");
- fout.close();
- }
- }
-
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
- }
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
+ if (mode == TestMode.local) {
+ FileWriter fstream = new FileWriter(inputFile);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write("us 2\n");
+ out.write("japan 2\n");
+ out.write("india 4\n");
+ out.write("us 2\n");
+ out.write("japan 1\n");
+ out.write("india 3\n");
+ out.write("nouse 5\n");
+ out.write("nowhere 4\n");
+ out.close();
+ }
+
+ if (mode == TestMode.cluster) {
+ FSDataOutputStream fout = fs.create(new Path (inputFile));
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 2\n");
+ fout.writeBytes("india 4\n");
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 1\n");
+ fout.writeBytes("india 3\n");
+ fout.writeBytes("nouse 5\n");
+ fout.writeBytes("nowhere 4\n");
+ fout.close();
}
-
}
public static void getTablePaths(String myMultiLocs) {
@@ -505,21 +366,15 @@ public class TestMultipleOutputs extends
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "word,count";
System.out.println("hello sort on word and count");
- String methodName = getCurrentMethodName();
+
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- }
+ String methodName = getCurrentMethodName();
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("india" + methodName).toString();
+ String str3 = getTableFullPath("japan" + methodName).toString();
+ myMultiLocs = str1 + "," + str2 + "," + str3;
+
getTablePaths(myMultiLocs);
System.out.println("strTable1: " + strTable1.toString());
System.out.println("strTable2: " + strTable2.toString());
@@ -529,7 +384,6 @@ public class TestMultipleOutputs extends
runMR(myMultiLocs, sortKey);
checkTable(myMultiLocs);
System.out.println("DONE test 1");
-
}
@Test
@@ -541,28 +395,21 @@ public class TestMultipleOutputs extends
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "word";
System.out.println("hello sort on word and count");
- String methodName = getCurrentMethodName();
+ String methodName = "test2";
String myMultiLocs = null;
- System.out.println("which cluster: " + whichCluster);
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- }
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("india" + methodName).toString();
+ String str3 = getTableFullPath("japan" + methodName).toString();
+ myMultiLocs = str1 + "," + str2 + "," + str3;
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(myMultiLocs, sortKey);
checkTable(myMultiLocs);
+ System.out.println("DONE test 2");
}
@Test
@@ -571,33 +418,23 @@ public class TestMultipleOutputs extends
/*
* test 'count' sort key
*/
- // setUpOnce();
System.out.println("******Start testcase: " + getCurrentMethodName());
sortKey = "count";
- System.out.println("hello sort on word and count, which cluster: "
- + whichCluster);
+
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "india"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "japan"
- + methodName);
- } else {
-
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "india"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "japan"
- + methodName);
- }
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("india" + methodName).toString();
+ String str3 = getTableFullPath("japan" + methodName).toString();
+ myMultiLocs = str1 + "," + str2 + "," + str3;
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
removeDir(new Path(strTable3));
runMR(myMultiLocs, sortKey);
checkTable(myMultiLocs);
+ System.out.println("DONE test 3");
}
static class MapClass extends
@@ -719,12 +556,10 @@ public class TestMultipleOutputs extends
return 0;
}
-
}
public void runMR(String myMultiLocs, String sortKey) throws ParseException,
IOException, Exception, org.apache.hadoop.zebra.parser.ParseException {
-
Job job = new Job(conf);
job.setJobName("TestMultipleOutputs");
job.setJarByClass(TestMultipleOutputs.class);
@@ -742,7 +577,6 @@ public class TestMultipleOutputs extends
//job.setNumMapTasks(1);
// output settings
-
job.setOutputFormatClass(BasicTableOutputFormat.class);
BasicTableOutputFormat.setMultipleOutputs(job, myMultiLocs,
TestMultipleOutputs.OutputPartitionerClass.class);
@@ -774,13 +608,10 @@ public class TestMultipleOutputs extends
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}
Modified: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2.java?rev=925988&r1=925987&r2=925988&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2.java (original)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/mapreduce/TestMultipleOutputs2.java Mon Mar 22 07:54:51 2010
@@ -30,12 +30,8 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -53,13 +49,11 @@ import org.apache.hadoop.zebra.schema.Sc
import org.apache.hadoop.zebra.types.TypesUtils;
import org.apache.hadoop.zebra.types.ZebraTuple;
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.data.Tuple;
-import org.apache.pig.test.MiniCluster;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.hadoop.zebra.BaseTestCase;
/**
* This is a sample a complete MR sample code for Table. It doens't contain
@@ -80,120 +74,21 @@ import org.junit.Test;
*
*
*/
-public class TestMultipleOutputs2 extends Configured implements Tool {
-
+public class TestMultipleOutputs2 extends BaseTestCase implements Tool {
static String inputPath;
static String inputFileName = "multi-input.txt";
- protected static ExecType execType = ExecType.LOCAL;
- private static MiniCluster cluster;
- protected static PigServer pigServer;
- // private static Path pathWorking, pathTable1, path2, path3,
- // pathTable4, pathTable5;
- private static Configuration conf;
public static String sortKey = null;
- private static FileSystem fs;
-
- private static String zebraJar;
- private static String whichCluster;
- private static String multiLocs;
private static String strTable1 = null;
private static String strTable2 = null;
- private static String strTable3 = null;
@BeforeClass
- public static void setUpOnce() throws IOException {
- if (System.getenv("hadoop.log.dir") == null) {
- String base = new File(".").getPath(); // getAbsolutePath();
- System
- .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
- }
-
- // by default we use miniCluster
- if (System.getenv("whichCluster") == null) {
- whichCluster = "miniCluster";
- } else {
- whichCluster = System.getenv("whichCluster");
- }
-
- if (conf == null) {
- conf = new Configuration();
- }
+ public static void setUpOnce() throws Exception {
+ init();
- if (whichCluster.equals("realCluster")) {
- System.out.println(" get env hadoop home: " + System.getenv("HADOOP_HOME"));
- System.out.println(" get env user name: " + System.getenv("USER"));
-
- if (System.getenv("HADOOP_HOME") == null) {
- System.out.println("Please set HADOOP_HOME for realCluster testing mode");
- System.exit(0);
- }
-
- if (System.getenv("USER") == null) {
- System.out.println("Please set USER for realCluster testing mode");
- System.exit(0);
- }
-
- zebraJar = System.getenv("HADOOP_HOME") + "/lib/zebra.jar";
+ inputPath = getTableFullPath(inputFileName).toString();
- File file = new File(zebraJar);
- if (!file.exists()) {
- System.out.println("Please place zebra.jar at $HADOOP_HOME/lib");
- System.exit(0);
- }
- }
-
- // set inputPath and output path
- String workingDir = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- inputPath = new String("/user/" + System.getenv("USER") + "/"
- + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + "," + "/user/" + System.getenv("USER") + "/" + "india" + ","
- + "/user/" + System.getenv("USER") + "/" + "japan");
- fs = new Path(inputPath).getFileSystem(conf);
-
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- inputPath = new String(workingDir + "/" + inputFileName);
- System.out.println("inputPath: " + inputPath);
- multiLocs = new String(workingDir + "/" + "us" + "," + workingDir + "/"
- + "india" + "," + workingDir + "/" + "japan");
- }
writeToFile(inputPath);
- // check inputPath existence
- File inputFile = new File(inputPath);
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("realCluster")) {
- System.out.println("Please put inputFile in hdfs: " + inputPath);
- // System.exit(0);
- }
- if (!inputFile.exists() && whichCluster.equalsIgnoreCase("miniCluster")) {
- System.out
- .println("Please put inputFile under workingdir. working dir is : "
- + workingDir);
- System.exit(0);
- }
-
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil
- .toProperties(conf));
- pigServer.registerJar(zebraJar);
-
- }
-
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
- if (execType == ExecType.MAPREDUCE) {
- cluster = MiniCluster.buildCluster();
- pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- fs = cluster.getFileSystem();
-
- } else {
- pigServer = new PigServer(ExecType.LOCAL);
- }
- }
}
public String getCurrentMethodName() {
@@ -220,73 +115,32 @@ public class TestMultipleOutputs2 extend
return methodName;
}
public static void writeToFile (String inputFile) throws IOException{
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- FileWriter fstream = new FileWriter(inputFile);
- BufferedWriter out = new BufferedWriter(fstream);
- out.write("us 2\n");
- out.write("japan 2\n");
- out.write("india 4\n");
- out.write("us 2\n");
- out.write("japan 1\n");
- out.write("india 3\n");
- out.write("nouse 5\n");
- out.write("nowhere 4\n");
- out.close();
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- FSDataOutputStream fout = fs.create(new Path (inputFile));
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 2\n");
- fout.writeBytes("india 4\n");
- fout.writeBytes("us 2\n");
- fout.writeBytes("japan 1\n");
- fout.writeBytes("india 3\n");
- fout.writeBytes("nouse 5\n");
- fout.writeBytes("nowhere 4\n");
- fout.close();
- }
- }
-
-
- public Path generateOutPath(String currentMethod) {
- Path outPath = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- outPath = new Path("/user/" + System.getenv("USER") + "/multiOutput/"
- + currentMethod);
- } else {
- String workingDir = fs.getWorkingDirectory().toString().split(":")[1];
- outPath = new Path(workingDir + "/multiOutput/" + currentMethod);
- System.out.println("output file: " + outPath.toString());
- }
- return outPath;
- }
-
- public void removeDir(Path outPath) throws IOException {
- String command = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- command = System.getenv("HADOOP_HOME") + "/bin/hadoop fs -rmr "
- + outPath.toString();
- } else {
- StringTokenizer st = new StringTokenizer(outPath.toString(), ":");
- int count = 0;
- String file = null;
- while (st.hasMoreElements()) {
- count++;
- String token = st.nextElement().toString();
- if (count == 2)
- file = token;
- }
- command = "rm -rf " + file;
+ if (mode == TestMode.local) {
+ FileWriter fstream = new FileWriter(inputFile);
+ BufferedWriter out = new BufferedWriter(fstream);
+ out.write("us 2\n");
+ out.write("japan 2\n");
+ out.write("india 4\n");
+ out.write("us 2\n");
+ out.write("japan 1\n");
+ out.write("india 3\n");
+ out.write("nouse 5\n");
+ out.write("nowhere 4\n");
+ out.close();
}
- Runtime runtime = Runtime.getRuntime();
- Process proc = runtime.exec(command);
- int exitVal = -1;
- try {
- exitVal = proc.waitFor();
- } catch (InterruptedException e) {
- System.err.println(e);
+
+ if (mode == TestMode.cluster) {
+ FSDataOutputStream fout = fs.create(new Path (inputFile));
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 2\n");
+ fout.writeBytes("india 4\n");
+ fout.writeBytes("us 2\n");
+ fout.writeBytes("japan 1\n");
+ fout.writeBytes("india 3\n");
+ fout.writeBytes("nouse 5\n");
+ fout.writeBytes("nowhere 4\n");
+ fout.close();
}
-
}
public static void getTablePaths(String myMultiLocs) {
@@ -300,16 +154,16 @@ public class TestMultipleOutputs2 extend
while (st.hasMoreElements()) {
count++;
String token = st.nextElement().toString();
- if (whichCluster.equalsIgnoreCase("miniCluster")) {
+ if (mode == TestMode.local) {
System.out.println("in mini, token: "+token);
if (count == 1)
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
- if (whichCluster.equalsIgnoreCase("realCluster")) {
+
+ //if (whichCluster.equalsIgnoreCase("realCluster")) {
+ if (mode == TestMode.cluster) {
System.out.println("in real, token: "+token);
//in real, token: /user/hadoopqa/ustest3
//note: no prefix file: in real cluster
@@ -317,33 +171,24 @@ public class TestMultipleOutputs2 extend
strTable1 = token;
if (count == 2)
strTable2 = token;
- if (count == 3)
- strTable3 = token;
}
-
}
}
- public static void checkTableExists(boolean expected, String strDir) throws IOException{
+ public static void checkTableExists(boolean expected, String strDir) throws IOException{
File theDir = null;
boolean actual = false;
- if (whichCluster.equalsIgnoreCase("miniCluster")){
- theDir = new File(strDir.split(":")[1]);
- actual = theDir.exists();
+
+ theDir = new File(strDir.split(":")[0]);
+ actual = fs.exists(new Path (theDir.toString()));
- }
- if (whichCluster.equalsIgnoreCase("realCluster")){
- theDir = new File(strDir.split(":")[0]);
- actual = fs.exists(new Path (theDir.toString()));
- }
System.out.println("the dir : "+ theDir.toString());
- //the dir : /user/hadoopqa/ustest3
- //the dir : /homes/<uid>/grid/multipleoutput/pig-table/contrib/zebra/ustest3
if (actual != expected){
Assert.fail("dir exists or not is different from what expected.");
}
- }
+ }
+
public static void checkTable(String myMultiLocs) throws IOException {
System.out.println("myMultiLocs:" + myMultiLocs);
System.out.println("sorgetTablePathst key:" + sortKey);
@@ -353,10 +198,10 @@ public class TestMultipleOutputs2 extend
String query2 = null;
if (strTable1 != null) {
-
query1 = "records1 = LOAD '" + strTable1
+ "' USING org.apache.hadoop.zebra.pig.TableLoader();";
}
+
if (strTable2 != null) {
query2 = "records2 = LOAD '" + strTable2
+ "' USING org.apache.hadoop.zebra.pig.TableLoader();";
@@ -495,7 +340,6 @@ public class TestMultipleOutputs2 extend
}// if query2 != null
}
-
@Test
public void test1() throws ParseException, IOException,
@@ -508,17 +352,11 @@ public class TestMultipleOutputs2 extend
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "others"
- + methodName );
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName );
- }
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("others" + methodName).toString();
+ myMultiLocs = str1 + "," + str2;
+
getTablePaths(myMultiLocs);
System.out.println("strTable1: "+strTable1);
removeDir(new Path(strTable1));
@@ -528,6 +366,7 @@ public class TestMultipleOutputs2 extend
System.out.println("DONE test" +getCurrentMethodName());
}
+
@Test
public void test2() throws ParseException, IOException,
org.apache.hadoop.zebra.parser.ParseException, Exception {
@@ -539,17 +378,11 @@ public class TestMultipleOutputs2 extend
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "others"
- + methodName );
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName );
- }
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("others" + methodName).toString();
+ myMultiLocs = str1 + "," + str2;
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
@@ -558,7 +391,8 @@ public class TestMultipleOutputs2 extend
System.out.println("DONE test" +getCurrentMethodName());
}
- @Test
+
+ @Test
public void test3() throws ParseException, IOException,
org.apache.hadoop.zebra.parser.ParseException, Exception {
/*
@@ -569,30 +403,25 @@ public class TestMultipleOutputs2 extend
System.out.println("hello sort on word and count");
String methodName = getCurrentMethodName();
String myMultiLocs = null;
- if (whichCluster.equalsIgnoreCase("realCluster")) {
- myMultiLocs = new String("/user/" + System.getenv("USER") + "/" + "us"
- + methodName + "," + "/user/" + System.getenv("USER") + "/" + "others"
- + methodName );
- } else {
- RawLocalFileSystem rawLFS = new RawLocalFileSystem();
- fs = new LocalFileSystem(rawLFS);
- myMultiLocs = new String(fs.getWorkingDirectory() + "/" + "us"
- + methodName + "," + fs.getWorkingDirectory() + "/" + "others"
- + methodName );
- }
+
+ String str1 = getTableFullPath("us" + methodName).toString();
+ String str2 = getTableFullPath("others" + methodName).toString();
+ myMultiLocs = str1 + "," + str2;
+
getTablePaths(myMultiLocs);
removeDir(new Path(strTable1));
removeDir(new Path(strTable2));
runMR(myMultiLocs, sortKey);
+ System.out.println("strTable1 = " + strTable1);
+ System.out.println("strTable2 = " + strTable2);
checkTableExists(true, strTable1);
checkTableExists(true, strTable2);
checkTable(myMultiLocs);
System.out.println("DONE test" +getCurrentMethodName());
-
}
- static class MapClass extends
+ static class MapClass extends
Mapper<LongWritable, Text, BytesWritable, Tuple> {
private BytesWritable bytesKey;
private Tuple tupleRow;
@@ -769,13 +598,10 @@ System.out.println("value0 : "+value.get
}
public static void main(String[] args) throws Exception {
- //XXX
- System.out.println("******************* this is new today");
-
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs2(), args);
-
+ System.out.println("PASS");
System.exit(res);
}
}