You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2009/05/04 10:04:21 UTC
svn commit: r771231 - in /hadoop/core/trunk: ./
src/test/org/apache/hadoop/test/
Author: sharad
Date: Mon May 4 08:04:21 2009
New Revision: 771231
URL: http://svn.apache.org/viewvc?rev=771231&view=rev
Log:
HADOOP-5217. Split AllTestDriver for core, hdfs and mapred.
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/test/CoreTestDriver.java
hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsTestDriver.java
hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsWithMRTestDriver.java
hadoop/core/trunk/src/test/org/apache/hadoop/test/MapredTestDriver.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771231&r1=771230&r2=771231&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 4 08:04:21 2009
@@ -287,6 +287,8 @@
HADOOP-5717. Create public enum class for the Framework counters in
org.apache.hadoop.mapreduce. (Amareshwari Sriramadasu via sharad)
+ HADOOP-5217. Split AllTestDriver for core, hdfs and mapred. (sharad)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java?rev=771231&r1=771230&r2=771231&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java Mon May 4 08:04:21 2009
@@ -19,69 +19,28 @@
package org.apache.hadoop.test;
import org.apache.hadoop.util.ProgramDriver;
-import org.apache.hadoop.mapred.BigMapOutput;
-import org.apache.hadoop.mapred.GenericMRLoadGenerator;
-import org.apache.hadoop.mapred.MRBench;
-import org.apache.hadoop.mapred.ReliabilityTest;
-import org.apache.hadoop.mapred.SortValidator;
-import org.apache.hadoop.mapred.TestMapRed;
-import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
-import org.apache.hadoop.mapred.TestTextInputFormat;
-import org.apache.hadoop.hdfs.BenchmarkThroughput;
-import org.apache.hadoop.hdfs.NNBench;
-import org.apache.hadoop.fs.DistributedFSCheck;
-import org.apache.hadoop.fs.TestDFSIO;
-import org.apache.hadoop.fs.DFSCIOTest;
-import org.apache.hadoop.fs.TestFileSystem;
-import org.apache.hadoop.io.FileBench;
-import org.apache.hadoop.io.TestArrayFile;
-import org.apache.hadoop.io.TestSequenceFile;
-import org.apache.hadoop.io.TestSetFile;
-import org.apache.hadoop.ipc.TestIPC;
-import org.apache.hadoop.ipc.TestRPC;
-import org.apache.hadoop.mapred.ThreadedMapBenchmark;
+
+@Deprecated
+//Class to be removed after the project split
public class AllTestDriver {
/**
* A description of the test program for running all the tests using jar file
*/
public static void main(String argv[]){
- ProgramDriver pgd = new ProgramDriver();
+ ProgramDriver pd = new ProgramDriver();
+ new CoreTestDriver(pd);
+ new HdfsTestDriver(pd);
+ new HdfsWithMRTestDriver(pd);
+ new MapredTestDriver(pd);
+
try {
- pgd.addClass("threadedmapbench", ThreadedMapBenchmark.class,
- "A map/reduce benchmark that compares the performance " +
- "of maps with multiple spills over maps with 1 spill");
- pgd.addClass("mrbench", MRBench.class, "A map/reduce benchmark that can create many small jobs");
- pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode.");
- pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
- pgd.addClass("testfilesystem", TestFileSystem.class, "A test for FileSystem read/write.");
- pgd.addClass("testsequencefile", TestSequenceFile.class, "A test for flat files of binary key value pairs.");
- pgd.addClass("testsetfile", TestSetFile.class, "A test for flat files of binary key/value pairs.");
- pgd.addClass("testarrayfile", TestArrayFile.class, "A test for flat files of binary key/value pairs.");
- pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
- pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
- pgd.addClass("testsequencefileinputformat", TestSequenceFileInputFormat.class, "A test for sequence file input format.");
- pgd.addClass("testtextinputformat", TestTextInputFormat.class, "A test for text input format.");
- pgd.addClass("TestDFSIO", TestDFSIO.class, "Distributed i/o benchmark.");
- pgd.addClass("DFSCIOTest", DFSCIOTest.class, "Distributed i/o benchmark of libhdfs.");
- pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, "Distributed checkup of the file system consistency.");
- pgd.addClass("testmapredsort", SortValidator.class,
- "A map/reduce program that validates the map-reduce framework's sort.");
- pgd.addClass("testbigmapoutput", BigMapOutput.class,
- "A map/reduce program that works on a very big " +
- "non-splittable file and does identity map/reduce");
- pgd.addClass("loadgen", GenericMRLoadGenerator.class, "Generic map/reduce load generator");
- pgd.addClass("filebench", FileBench.class, "Benchmark SequenceFile(Input|Output)Format (block,record compressed and uncompressed), Text(Input|Output)Format (compressed and uncompressed)");
- pgd.addClass("dfsthroughput", BenchmarkThroughput.class,
- "measure hdfs throughput");
- pgd.addClass("MRReliabilityTest", ReliabilityTest.class,
- "A program that tests the reliability of the MR framework by " +
- "injecting faults/failures");
- pgd.driver(argv);
- } catch(Throwable e) {
+ pd.driver(argv);
+ } catch (Throwable e) {
e.printStackTrace();
}
}
+
}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/test/CoreTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/test/CoreTestDriver.java?rev=771231&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/test/CoreTestDriver.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/test/CoreTestDriver.java Mon May 4 08:04:21 2009
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.io.TestArrayFile;
+import org.apache.hadoop.io.TestSequenceFile;
+import org.apache.hadoop.io.TestSetFile;
+import org.apache.hadoop.ipc.TestIPC;
+import org.apache.hadoop.ipc.TestRPC;
+import org.apache.hadoop.util.ProgramDriver;
+
+/**
+ * Driver for core tests.
+ */
+public class CoreTestDriver {
+
+ private ProgramDriver pgd;
+
+ public CoreTestDriver() {
+ this(new ProgramDriver());
+ }
+
+ public CoreTestDriver(ProgramDriver pgd) {
+ this.pgd = pgd;
+ try {
+ pgd.addClass("testsequencefile", TestSequenceFile.class,
+ "A test for flat files of binary key value pairs.");
+ pgd.addClass("testsetfile", TestSetFile.class,
+ "A test for flat files of binary key/value pairs.");
+ pgd.addClass("testarrayfile", TestArrayFile.class,
+ "A test for flat files of binary key/value pairs.");
+ pgd.addClass("testrpc", TestRPC.class, "A test for rpc.");
+ pgd.addClass("testipc", TestIPC.class, "A test for ipc.");
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void run(String argv[]) {
+ try {
+ pgd.driver(argv);
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String argv[]){
+ new CoreTestDriver().run(argv);
+ }
+}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsTestDriver.java?rev=771231&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsTestDriver.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsTestDriver.java Mon May 4 08:04:21 2009
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.hdfs.BenchmarkThroughput;
+import org.apache.hadoop.util.ProgramDriver;
+
+/**
+ * Driver for HDFS tests. The tests should NOT depend on map-reduce APIs.
+ */
+public class HdfsTestDriver {
+
+ private ProgramDriver pgd;
+
+ public HdfsTestDriver() {
+ this(new ProgramDriver());
+ }
+
+ public HdfsTestDriver(ProgramDriver pgd) {
+ this.pgd = pgd;
+ try {
+ pgd.addClass("dfsthroughput", BenchmarkThroughput.class,
+ "measure hdfs throughput");
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void run(String argv[]) {
+ try {
+ pgd.driver(argv);
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String argv[]){
+ new HdfsTestDriver().run(argv);
+ }
+}
\ No newline at end of file
Added: hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsWithMRTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsWithMRTestDriver.java?rev=771231&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsWithMRTestDriver.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/test/HdfsWithMRTestDriver.java Mon May 4 08:04:21 2009
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.fs.DFSCIOTest;
+import org.apache.hadoop.fs.DistributedFSCheck;
+import org.apache.hadoop.fs.TestDFSIO;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.hdfs.NNBench;
+import org.apache.hadoop.io.FileBench;
+import org.apache.hadoop.util.ProgramDriver;
+
+/*
+ * Driver for HDFS tests, which require map-reduce to run.
+ */
+public class HdfsWithMRTestDriver {
+
+
+ private ProgramDriver pgd;
+
+ public HdfsWithMRTestDriver() {
+ this(new ProgramDriver());
+ }
+
+ public HdfsWithMRTestDriver(ProgramDriver pgd) {
+ this.pgd = pgd;
+ try {
+ pgd.addClass("nnbench", NNBench.class,
+ "A benchmark that stresses the namenode.");
+ pgd.addClass("testfilesystem", TestFileSystem.class,
+ "A test for FileSystem read/write.");
+ pgd.addClass("TestDFSIO", TestDFSIO.class,
+ "Distributed i/o benchmark.");
+ pgd.addClass("DFSCIOTest", DFSCIOTest.class, "" +
+ "Distributed i/o benchmark of libhdfs.");
+ pgd.addClass("DistributedFSCheck", DistributedFSCheck.class,
+ "Distributed checkup of the file system consistency.");
+ pgd.addClass("filebench", FileBench.class,
+ "Benchmark SequenceFile(Input|Output)Format " +
+ "(block,record compressed and uncompressed), " +
+ "Text(Input|Output)Format (compressed and uncompressed)");
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void run(String argv[]) {
+ try {
+ pgd.driver(argv);
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String argv[]){
+ new HdfsWithMRTestDriver().run(argv);
+ }
+}
+
Added: hadoop/core/trunk/src/test/org/apache/hadoop/test/MapredTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/test/MapredTestDriver.java?rev=771231&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/test/MapredTestDriver.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/test/MapredTestDriver.java Mon May 4 08:04:21 2009
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.test;
+
+import org.apache.hadoop.mapred.BigMapOutput;
+import org.apache.hadoop.mapred.GenericMRLoadGenerator;
+import org.apache.hadoop.mapred.MRBench;
+import org.apache.hadoop.mapred.ReliabilityTest;
+import org.apache.hadoop.mapred.SortValidator;
+import org.apache.hadoop.mapred.TestMapRed;
+import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
+import org.apache.hadoop.mapred.TestTextInputFormat;
+import org.apache.hadoop.mapred.ThreadedMapBenchmark;
+import org.apache.hadoop.util.ProgramDriver;
+
+/**
+ * Driver for Map-reduce tests.
+ *
+ */
+public class MapredTestDriver {
+
+ private ProgramDriver pgd;
+
+ public MapredTestDriver() {
+ this(new ProgramDriver());
+ }
+
+ public MapredTestDriver(ProgramDriver pgd) {
+ this.pgd = pgd;
+ try {
+ pgd.addClass("threadedmapbench", ThreadedMapBenchmark.class,
+ "A map/reduce benchmark that compares the performance " +
+ "of maps with multiple spills over maps with 1 spill");
+ pgd.addClass("mrbench", MRBench.class,
+ "A map/reduce benchmark that can create many small jobs");
+ pgd.addClass("mapredtest", TestMapRed.class, "A map/reduce test check.");
+ pgd.addClass("testsequencefileinputformat",
+ TestSequenceFileInputFormat.class,
+ "A test for sequence file input format.");
+ pgd.addClass("testtextinputformat", TestTextInputFormat.class,
+ "A test for text input format.");
+ pgd.addClass("testmapredsort", SortValidator.class,
+ "A map/reduce program that validates the " +
+ "map-reduce framework's sort.");
+ pgd.addClass("testbigmapoutput", BigMapOutput.class,
+ "A map/reduce program that works on a very big " +
+ "non-splittable file and does identity map/reduce");
+ pgd.addClass("loadgen", GenericMRLoadGenerator.class,
+ "Generic map/reduce load generator");
+ pgd.addClass("MRReliabilityTest", ReliabilityTest.class,
+ "A program that tests the reliability of the MR framework by " +
+ "injecting faults/failures");
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void run(String argv[]) {
+ try {
+ pgd.driver(argv);
+ } catch(Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String argv[]){
+ new MapredTestDriver().run(argv);
+ }
+}
+