You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/05/03 01:07:35 UTC
svn commit: r399077 - in /lucene/hadoop/trunk: ./
src/examples/org/apache/hadoop/examples/ src/java/org/apache/hadoop/util/
src/test/org/apache/hadoop/fs/ src/test/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/test/
Author: cutting
Date: Tue May 2 16:07:32 2006
New Revision: 399077
URL: http://svn.apache.org/viewcvs?rev=399077&view=rev
Log:
HADOOP-184. Restructure some test code to better support testing on a cluster. Contributed by Mahadev.
Added:
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java
- copied, changed from r399074, lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/test/
lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java
Removed:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/build.xml
lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/CHANGES.txt?rev=399077&r1=399076&r2=399077&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue May 2 16:07:32 2006
@@ -158,6 +158,9 @@
41. HADOOP-190. If a child process hangs after it has reported
completion, its output should not be lost. (Stack via cutting)
+42. HADOOP-184. Re-structure some test code to better support testing
+ on a cluster. (Mahadev Konar via cutting)
+
Release 0.1.1 - 2006-04-08
Modified: lucene/hadoop/trunk/build.xml
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/build.xml?rev=399077&r1=399076&r2=399077&view=diff
==============================================================================
--- lucene/hadoop/trunk/build.xml (original)
+++ lucene/hadoop/trunk/build.xml Tue May 2 16:07:32 2006
@@ -182,7 +182,7 @@
<!-- ================================================================== -->
<!-- Compile test code -->
<!-- ================================================================== -->
- <target name="compile-test" depends="compile">
+ <target name="compile-test" depends="compile, jar">
<javac
encoding="${build.encoding}"
srcdir="${test.src.dir}"
@@ -195,6 +195,13 @@
deprecation="${javac.deprecation}">
<classpath refid="test.classpath"/>
</javac>
+ <jar jarfile="${build.dir}/${final.name}-test.jar"
+ basedir="${test.build.classes}">
+ <manifest>
+ <attribute name="Main-Class"
+ value="org/apache/hadoop/test/AllTestDriver"/>
+ </manifest>
+ </jar>
</target>
<!-- ================================================================== -->
@@ -290,6 +297,8 @@
<copy file="${build.dir}/${final.name}.jar" todir="${dist.dir}"/>
<copy file="${build.dir}/${final.name}-examples.jar" todir="${dist.dir}"/>
+
+ <copy file="${build.dir}/${final.name}-test.jar" todir="${dist.dir}"/>
<copy todir="${dist.dir}/bin">
<fileset dir="bin"/>
Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java?rev=399077&r1=399076&r2=399077&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java (original)
+++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/ExampleDriver.java Tue May 2 16:07:32 2006
@@ -15,7 +15,7 @@
*/
package org.apache.hadoop.examples;
-
+import org.apache.hadoop.util.ProgramDriver;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.TreeMap;
@@ -29,107 +29,21 @@
* A description of an example program based on its class and a
* human-readable description.
* @author Owen O'Malley
- * @date feb 2006
+ * @date april 2006
*/
- static private class ProgramDescription {
-
- static final Class[] paramTypes = new Class[] {String[].class};
-
- /**
- * Create a description of an example program.
- * @param mainClass the class with the main for the example program
- * @param description a string to display to the user in help messages
- * @throws SecurityException if we can't use reflection
- * @throws NoSuchMethodException if the class doesn't have a main method
- */
- public ProgramDescription(Class mainClass,
- String description)
- throws SecurityException, NoSuchMethodException {
- this.main = mainClass.getMethod("main", paramTypes);
- this.description = description;
- }
-
- /**
- * Invoke the example application with the given arguments
- * @param args the arguments for the application
- * @throws Throwable The exception thrown by the invoked method
- */
- public void invoke(String[] args)
- throws Throwable {
- try {
- main.invoke(null, new Object[]{args});
- } catch (InvocationTargetException except) {
- throw except.getCause();
- }
- }
- public String getDescription() {
- return description;
+ public static void main(String argv[]){
+ ProgramDriver pgd = new ProgramDriver();
+ try {
+ pgd.addClass("wordcount", WordCount.class,
+ "A map/reduce program that counts the words in the input files.");
+ pgd.addClass("grep", Grep.class,
+ "A map/reduce program that counts the matches of a regex in the input.");
+ pgd.driver(argv);
+ }
+ catch(Throwable e){
+ e.printStackTrace();
+ }
}
-
- private Method main;
- private String description;
- }
-
- private static void printUsage(Map programs) {
- System.out.println("Valid program names are:");
- for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) {
- Map.Entry item = (Entry) itr.next();
- System.out.println(" " + (String) item.getKey() + ": " +
- ((ProgramDescription) item.getValue()).getDescription());
- }
- }
-
- /**
- * This is a driver for the example programs.
- * It looks at the first command line argument and tries to find an
- * example program with that name.
- * If it is found, it calls the main method in that class with the rest
- * of the command line arguments.
- * @param args The argument from the user. args[0] is the command to run.
- * @throws NoSuchMethodException
- * @throws SecurityException
- * @throws IllegalAccessException
- * @throws IllegalArgumentException
- * @throws Throwable Anything thrown by the example program's main
- */
- public static void main(String[] args)
- throws Throwable
- {
- Map programs = new TreeMap();
-
- // Add new programs to this list
- programs.put("wordcount", new ProgramDescription(WordCount.class,
- "A map/reduce program that counts the words in the input files."));
- programs.put("grep", new ProgramDescription(Grep.class,
- "A map/reduce program that counts the matches of a regex in the input."));
- programs.put("sort", new ProgramDescription(Sort.class,
- "Sort binary keys and values."));
- programs.put("writer", new ProgramDescription(RandomWriter.class,
- "Write random binary key/value pairs"));
-
- // Make sure they gave us a program name.
- if (args.length == 0) {
- System.out.println("An example program must be given as the" +
- " first argument.");
- printUsage(programs);
- return;
- }
-
- // And that it is good.
- ProgramDescription pgm = (ProgramDescription) programs.get(args[0]);
- if (pgm == null) {
- System.out.println("Unknown program '" + args[0] + "' chosen.");
- printUsage(programs);
- return;
- }
-
- // Remove the leading argument and call main
- String[] new_args = new String[args.length - 1];
- for(int i=1; i < args.length; ++i) {
- new_args[i-1] = args[i];
- }
- pgm.invoke(new_args);
- }
-
}
+
Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java?rev=399077&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/ProgramDriver.java Tue May 2 16:07:32 2006
@@ -0,0 +1,144 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.TreeMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** A driver that is used to run programs added to it
+ */
+
+public class ProgramDriver {
+
+ /**
+ * A description of a program based on its class and a
+ * human-readable description.
+ * @author Owen O'Malley
+ * @date april 2006
+ */
+ Map programs;
+
+ public ProgramDriver(){
+ programs = new TreeMap();
+ }
+
+ static private class ProgramDescription {
+
+ static final Class[] paramTypes = new Class[] {String[].class};
+
+ /**
+ * Create a description of an example program.
+ * @param mainClass the class with the main for the example program
+ * @param description a string to display to the user in help messages
+ * @throws SecurityException if we can't use reflection
+ * @throws NoSuchMethodException if the class doesn't have a main method
+ */
+ public ProgramDescription(Class mainClass,
+ String description)
+ throws SecurityException, NoSuchMethodException {
+ this.main = mainClass.getMethod("main", paramTypes);
+ this.description = description;
+ }
+
+ /**
+ * Invoke the example application with the given arguments
+ * @param args the arguments for the application
+ * @throws Throwable The exception thrown by the invoked method
+ */
+ public void invoke(String[] args)
+ throws Throwable {
+ try {
+ main.invoke(null, new Object[]{args});
+ } catch (InvocationTargetException except) {
+ throw except.getCause();
+ }
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ private Method main;
+ private String description;
+ }
+
+ private static void printUsage(Map programs) {
+ System.out.println("Valid program names are:");
+ for(Iterator itr=programs.entrySet().iterator(); itr.hasNext();) {
+ Map.Entry item = (Entry) itr.next();
+ System.out.println(" " + (String) item.getKey() + ": " +
+ ((ProgramDescription) item.getValue()).getDescription());
+ }
+ }
+
+ /**
+ * This is the method that adds the classed to the repository
+ * @param name The name of the string you want the class instance to be called with
+ * @param mainClass The class that you want to add to the repository
+ * @param description The description of the class
+ * @throws NoSuchMethodException
+ * @throws SecurityException
+ */
+ public void addClass (String name, Class mainClass, String description) throws Throwable {
+ programs.put(name , new ProgramDescription(mainClass, description));
+ }
+
+ /**
+ * This is a driver for the example programs.
+ * It looks at the first command line argument and tries to find an
+ * example program with that name.
+ * If it is found, it calls the main method in that class with the rest
+ * of the command line arguments.
+ * @param args The argument from the user. args[0] is the command to run.
+ * @throws NoSuchMethodException
+ * @throws SecurityException
+ * @throws IllegalAccessException
+ * @throws IllegalArgumentException
+ * @throws Throwable Anything thrown by the example program's main
+ */
+ public void driver(String[] args)
+ throws Throwable
+ {
+ // Make sure they gave us a program name.
+ if (args.length == 0) {
+ System.out.println("An example program must be given as the" +
+ " first argument.");
+ printUsage(programs);
+ return;
+ }
+
+ // And that it is good.
+ ProgramDescription pgm = (ProgramDescription) programs.get(args[0]);
+ if (pgm == null) {
+ System.out.println("Unknown program '" + args[0] + "' chosen.");
+ printUsage(programs);
+ return;
+ }
+
+ // Remove the leading argument and call main
+ String[] new_args = new String[args.length - 1];
+ for(int i=1; i < args.length; ++i) {
+ new_args[i-1] = args[i];
+ }
+ pgm.invoke(new_args);
+ }
+
+}
Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=399077&r1=399076&r2=399077&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/fs/TestFileSystem.java Tue May 2 16:07:32 2006
@@ -59,6 +59,10 @@
writeTest(fs, false);
readTest(fs, false);
seekTest(fs, false);
+ fs.delete(CONTROL_DIR);
+ fs.delete(DATA_DIR);
+ fs.delete(WRITE_DIR);
+ fs.delete(READ_DIR);
}
public static void createControlFile(FileSystem fs,
Copied: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java (from r399074, lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java)
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java?p2=lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java&p1=lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java&r1=399074&r2=399077&rev=399077&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/MapredLoadTest.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/TestMapRed.java Tue May 2 16:07:32 2006
@@ -18,10 +18,11 @@
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.conf.*;
-
+import junit.framework.TestCase;
import java.io.*;
import java.util.*;
+
/**********************************************************
* MapredLoadTest generates a bunch of work that exercises
* a Hadoop Map-Reduce system (and DFS, too). It goes through
@@ -55,8 +56,9 @@
* 7) A mapred job integrates all the count files into a single one.
*
**********************************************************/
-public class MapredLoadTest {
+public class TestMapRed extends TestCase {
/**
+ * Modified to make it a junit test.
* The RandomGen Job does the actual work of creating
* a huge file of assorted numbers. It receives instructions
* as to how many times each number should be counted. Then
@@ -197,24 +199,27 @@
}
}
- int range;
- int counts;
- Random r = new Random();
- Configuration conf;
+ private static int range = 10;
+ private static int counts = 100;
+ private static Random r = new Random();
+ private static Configuration conf = new Configuration();
/**
- * MapredLoadTest
- */
- public MapredLoadTest(int range, int counts, Configuration conf) throws IOException {
- this.range = range;
- this.counts = counts;
- this.conf = conf;
+ public TestMapRed(int range, int counts, Configuration conf) throws IOException {
+ this.range = range;
+ this.counts = counts;
+ this.conf = conf;
+ }
+ **/
+
+ public void testMapred() throws Exception {
+ launch();
}
/**
*
*/
- public void launch() throws IOException {
+ public static void launch() throws Exception {
//
// Generate distribution of ints. This is the answer key.
//
@@ -268,7 +273,7 @@
// file of random numbers.
//
Path randomOuts = new Path(testdir, "genouts");
- fs.mkdirs(randomOuts);
+ //fs.mkdirs(randomOuts);
JobConf genJob = new JobConf(conf);
@@ -317,7 +322,7 @@
//
int intermediateReduces = 10;
Path intermediateOuts = new Path(testdir, "intermediateouts");
- fs.mkdirs(intermediateOuts);
+ //fs.mkdirs(intermediateOuts);
JobConf checkJob = new JobConf(conf);
checkJob.setInputPath(randomOuts);
checkJob.setInputKeyClass(LongWritable.class);
@@ -342,7 +347,7 @@
// all the files.
//
Path finalOuts = new Path(testdir, "finalouts");
- fs.mkdirs(finalOuts);
+ //fs.mkdirs(finalOuts);
JobConf mergeJob = new JobConf(conf);
mergeJob.setInputPath(intermediateOuts);
mergeJob.setInputKeyClass(IntWritable.class);
@@ -415,6 +420,7 @@
} finally {
bw.close();
}
+ fs.delete(testdir);
}
/**
@@ -422,7 +428,7 @@
*/
public static void main(String[] argv) throws Exception {
if (argv.length < 2) {
- System.err.println("Usage: MapredLoadTest <range> <counts>");
+ System.err.println("Usage: TestMapRed <range> <counts>");
System.err.println();
System.err.println("Note: a good test will have a <counts> value that is substantially larger than the <range>");
return;
@@ -431,8 +437,6 @@
int i = 0;
int range = Integer.parseInt(argv[i++]);
int counts = Integer.parseInt(argv[i++]);
-
- MapredLoadTest mlt = new MapredLoadTest(range, counts, new Configuration());
- mlt.launch();
+ launch();
}
}
Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java?rev=399077&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java Tue May 2 16:07:32 2006
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.util.ProgramDriver;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.TreeMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.hadoop.mapred.TestMapRed;
+import org.apache.hadoop.mapred.TestTextInputFormat;
+import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
+import org.apache.hadoop.dfs.ClusterTestDFS;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.io.TestArrayFile;
+import org.apache.hadoop.io.TestSetFile;
+import org.apache.hadoop.io.TestSequenceFile;
+import org.apache.hadoop.ipc.TestIPC;
+import org.apache.hadoop.ipc.TestRPC;
+
+public class AllTestDriver {
+
+ /**
+ * A description of the test program for running all the tests using jar file
+ * @date April 2006
+ */
+
+ public static void main(String argv[]){
+ ProgramDriver pgd = new ProgramDriver();
+ try {
+ pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
+ pgd.addClass("clustertestdfs", ClusterTestDFS.class, "A pseudo distributed test for DFS.");
+ pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
+ pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
+ pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
+ pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
+ pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
+ pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
+ pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
+ pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
+ pgd.driver(argv);
+ }
+ catch(Throwable e){
+ e.printStackTrace();
+ }
+ }
+}