You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2009/05/05 11:16:52 UTC
svn commit: r771635 [1/4] - in /hadoop/core/trunk: ./
src/test/org/apache/hadoop/cli/ src/test/org/apache/hadoop/cli/util/
src/test/org/apache/hadoop/hdfs/server/namenode/
Author: sharad
Date: Tue May 5 09:16:51 2009
New Revision: 771635
URL: http://svn.apache.org/viewvc?rev=771635&view=rev
Log:
HADOOP-5081. Split TestCLI into HDFS, Mapred and Core tests.
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java
hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java
hadoop/core/trunk/src/test/org/apache/hadoop/cli/testHDFSConf.xml
hadoop/core/trunk/src/test/org/apache/hadoop/cli/testMRConf.xml
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/build.xml
hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/CommandExecutor.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May 5 09:16:51 2009
@@ -312,6 +312,8 @@
number of fields to be sufficient to construct the select query. (Enis
Soztutar via cdouglas)
+ HADOOP-5081. Split TestCLI into HDFS, Mapred and Core tests. (sharad)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified: hadoop/core/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/build.xml?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/build.xml (original)
+++ hadoop/core/trunk/build.xml Tue May 5 09:16:51 2009
@@ -697,6 +697,8 @@
<copy file="${test.src.dir}/org/apache/hadoop/hdfs/hadoop-14-dfs-dir.tgz" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/hdfs/hadoop-dfs-dir.txt" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/testConf.xml" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/cli/testHDFSConf.xml" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/org/apache/hadoop/cli/testMRConf.xml" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data15bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data30bytes" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/org/apache/hadoop/cli/clitest_data/data60bytes" todir="${test.cache.data}"/>
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java?rev=771635&r1=771634&r2=771635&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java Tue May 5 09:16:51 2009
@@ -34,12 +34,8 @@
import org.apache.hadoop.cli.util.ComparatorData;
import org.apache.hadoop.cli.util.CLITestData.TestCmd;
import org.apache.hadoop.cli.util.CLITestData.TestCmd.CommandType;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.security.authorize.HadoopPolicyProvider;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
@@ -68,29 +64,22 @@
//By default, run the tests. The other mode is to run the commands and not
// compare the output
- public static String testMode = TESTMODE_TEST;
+ protected String testMode = TESTMODE_TEST;
// Storage for tests read in from the config file
- static ArrayList<CLITestData> testsFromConfigFile = null;
- static ArrayList<ComparatorData> testComparators = null;
- static String testConfigFile = "testConf.xml";
- String thisTestCaseName = null;
- static ComparatorData comparatorData = null;
-
- private static Configuration conf = null;
- private static MiniDFSCluster dfsCluster = null;
- private static DistributedFileSystem dfs = null;
- private static MiniMRCluster mrCluster = null;
- private static String namenode = null;
- private static String jobtracker = null;
- private static String clitestDataDir = null;
- private static String username = null;
+ protected ArrayList<CLITestData> testsFromConfigFile = null;
+ protected ArrayList<ComparatorData> testComparators = null;
+ protected String thisTestCaseName = null;
+ protected ComparatorData comparatorData = null;
+ protected Configuration conf = null;
+ protected String clitestDataDir = null;
+ protected String username = null;
/**
* Read the test config file - testConfig.xml
*/
- private void readTestConfigFile() {
-
+ protected void readTestConfigFile() {
+ String testConfigFile = getTestFile();
if (testsFromConfigFile == null) {
boolean success = false;
testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
@@ -106,6 +95,10 @@
}
}
+ protected String getTestFile() {
+ return "testConf.xml";
+ }
+
/*
* Setup
*/
@@ -113,60 +106,20 @@
// Read the testConfig.xml file
readTestConfigFile();
- // Start up the mini dfs cluster
- boolean success = false;
conf = new Configuration();
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG,
HadoopPolicyProvider.class, PolicyProvider.class);
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG,
true);
- // Many of the tests expect a replication value of 1 in the output
- conf.setInt("dfs.replication", 1);
-
- // Build racks and hosts configuration to test dfsAdmin -printTopology
- String [] racks = {"/rack1", "/rack1", "/rack2", "/rack2",
- "/rack2", "/rack3", "/rack4", "/rack4" };
- String [] hosts = {"host1", "host2", "host3", "host4",
- "host5", "host6", "host7", "host8" };
-
- dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
-
- namenode = conf.get("fs.default.name", "file:///");
clitestDataDir = new File(TEST_CACHE_DATA_DIR).
- toURI().toString().replace(' ', '+');
- username = System.getProperty("user.name");
-
- FileSystem fs = dfsCluster.getFileSystem();
- assertTrue("Not a HDFS: "+fs.getUri(),
- fs instanceof DistributedFileSystem);
- dfs = (DistributedFileSystem) fs;
-
- // Start up mini mr cluster
- JobConf mrConf = new JobConf(conf);
- mrCluster = new MiniMRCluster(1, dfsCluster.getFileSystem().getUri().toString(), 1,
- null, null, mrConf);
- jobtracker = mrCluster.createJobConf().get("mapred.job.tracker", "local");
-
- success = true;
-
- assertTrue("Error setting up Mini DFS & MR clusters", success);
+ toURI().toString().replace(' ', '+');
}
/**
* Tear down
*/
public void tearDown() throws Exception {
- boolean success = false;
- mrCluster.shutdown();
-
- dfs.close();
- dfsCluster.shutdown();
- success = true;
- Thread.sleep(2000);
-
- assertTrue("Error tearing down Mini DFS & MR clusters", success);
-
displayResults();
}
@@ -175,10 +128,8 @@
* @param cmd
* @return String expanded command
*/
- private String expandCommand(final String cmd) {
+ protected String expandCommand(final String cmd) {
String expCmd = cmd;
- expCmd = expCmd.replaceAll("NAMENODE", namenode);
- expCmd = expCmd.replaceAll("JOBTRACKER", jobtracker);
expCmd = expCmd.replaceAll("CLITEST_DATA", clitestDataDir);
expCmd = expCmd.replaceAll("USERNAME", username);
@@ -259,10 +210,18 @@
LOG.info("");
LOG.info(" Overall result: " +
(overallResults ? "+++ PASS +++" : "--- FAIL ---"));
- LOG.info(" # Tests pass: " + totalPass +
- " (" + (100 * totalPass / (totalPass + totalFail)) + "%)");
- LOG.info(" # Tests fail: " + totalFail +
- " (" + (100 * totalFail / (totalPass + totalFail)) + "%)");
+ if ((totalPass + totalFail) == 0) {
+ LOG.info(" # Tests pass: " + 0);
+ LOG.info(" # Tests fail: " + 0);
+ }
+ else
+ {
+ LOG.info(" # Tests pass: " + totalPass +
+ " (" + (100 * totalPass / (totalPass + totalFail)) + "%)");
+ LOG.info(" # Tests fail: " + totalFail +
+ " (" + (100 * totalFail / (totalPass + totalFail)) + "%)");
+ }
+
LOG.info(" # Validations done: " + totalComparators +
" (each test may do multiple validations)");
@@ -310,7 +269,7 @@
* @param compdata
* @return
*/
- private boolean compareTestOutput(ComparatorData compdata) {
+ private boolean compareTestOutput(ComparatorData compdata, Result cmdResult) {
// Compare the output based on the comparator
String comparatorType = compdata.getComparatorType();
Class<?> comparatorClass = null;
@@ -327,7 +286,7 @@
comparatorClass = Class.forName("org.apache.hadoop.cli.util." +
comparatorType);
ComparatorBase comp = (ComparatorBase) comparatorClass.newInstance();
- compareOutput = comp.compare(CommandExecutor.getLastCommandOutput(),
+ compareOutput = comp.compare(cmdResult.getCommandOutput(),
compdata.getExpectedOutput());
} catch (Exception e) {
LOG.info("Error in instantiating the comparator" + e);
@@ -351,9 +310,10 @@
// Execute the test commands
ArrayList<TestCmd> testCommands = testdata.getTestCommands();
+ Result cmdResult = null;
for (TestCmd cmd : testCommands) {
try {
- CommandExecutor.executeCommand(cmd, namenode, jobtracker);
+ cmdResult = execute(cmd);
} catch (Exception e) {
fail(StringUtils.stringifyException(e));
}
@@ -368,12 +328,12 @@
boolean compareOutput = false;
if (! comptype.equalsIgnoreCase("none")) {
- compareOutput = compareTestOutput(cd);
+ compareOutput = compareTestOutput(cd, cmdResult);
overallTCResult &= compareOutput;
}
- cd.setExitCode(CommandExecutor.getLastExitCode());
- cd.setActualOutput(CommandExecutor.getLastCommandOutput());
+ cd.setExitCode(cmdResult.getExitCode());
+ cd.setActualOutput(cmdResult.getCommandOutput());
cd.setTestResult(compareOutput);
}
testdata.setTestResult(overallTCResult);
@@ -382,7 +342,7 @@
ArrayList<TestCmd> cleanupCommands = testdata.getCleanupCommands();
for (TestCmd cmd : cleanupCommands) {
try {
- CommandExecutor.executeCommand(cmd, namenode, jobtracker);
+ execute(cmd);
} catch (Exception e) {
fail(StringUtils.stringifyException(e));
}
@@ -390,10 +350,14 @@
}
}
+ protected CommandExecutor.Result execute(TestCmd cmd) throws Exception {
+ throw new Exception("Unknow type of Test command:"+ cmd.getType());
+ }
+
/*
* Parser class for the test config xml file
*/
- static class TestConfigFileParser extends DefaultHandler {
+ class TestConfigFileParser extends DefaultHandler {
String charString = null;
CLITestData td = null;
ArrayList<TestCmd> testCommands = null;
Added: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java?rev=771635&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestHDFSCLI.java Tue May 5 09:16:51 2009
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CommandExecutor;
+import org.apache.hadoop.cli.util.CLITestData.TestCmd;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.util.ToolRunner;
+
+public class TestHDFSCLI extends TestCLI{
+
+ protected MiniDFSCluster dfsCluster = null;
+ protected DistributedFileSystem dfs = null;
+ protected String namenode = null;
+ protected DFSAdminCmdExecutor dfsAdmCmdExecutor = null;
+ protected FSCmdExecutor fsCmdExecutor = null;
+
+ public void setUp() throws Exception {
+ super.setUp();
+ // Many of the tests expect a replication value of 1 in the output
+ conf.setInt("dfs.replication", 1);
+
+ // Build racks and hosts configuration to test dfsAdmin -printTopology
+ String [] racks = {"/rack1", "/rack1", "/rack2", "/rack2",
+ "/rack2", "/rack3", "/rack4", "/rack4" };
+ String [] hosts = {"host1", "host2", "host3", "host4",
+ "host5", "host6", "host7", "host8" };
+ dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
+
+ namenode = conf.get("fs.default.name", "file:///");
+
+ username = System.getProperty("user.name");
+ dfsAdmCmdExecutor = new DFSAdminCmdExecutor(namenode);
+ fsCmdExecutor = new FSCmdExecutor(namenode);
+
+ FileSystem fs = dfsCluster.getFileSystem();
+ assertTrue("Not a HDFS: "+fs.getUri(),
+ fs instanceof DistributedFileSystem);
+ dfs = (DistributedFileSystem) fs;
+ }
+
+ protected String getTestFile() {
+ return "testHDFSConf.xml";
+ }
+
+ public void tearDown() throws Exception {
+ dfs.close();
+ dfsCluster.shutdown();
+ Thread.sleep(2000);
+ super.tearDown();
+ }
+
+ protected String expandCommand(final String cmd) {
+ String expCmd = cmd;
+ expCmd = expCmd.replaceAll("NAMENODE", namenode);
+ expCmd = super.expandCommand(cmd);
+ return expCmd;
+ }
+
+ protected Result execute(TestCmd cmd) throws Exception {
+ CommandExecutor executor = null;
+ switch(cmd.getType()) {
+ case DFSADMIN:
+ executor = dfsAdmCmdExecutor;
+ break;
+ case FS:
+ executor = fsCmdExecutor;
+ break;
+ default:
+ throw new Exception("Unknow type of Test command:"+ cmd.getType());
+ }
+ return executor.executeCommand(cmd.getCmd());
+ }
+
+ public static class DFSAdminCmdExecutor extends CommandExecutor {
+ private String namenode = null;
+ public DFSAdminCmdExecutor(String namenode) {
+ this.namenode = namenode;
+ }
+
+ protected void execute(final String cmd) throws Exception{
+ DFSAdmin shell = new DFSAdmin();
+ String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
+ ToolRunner.run(shell, args);
+ }
+ }
+
+ public static class FSCmdExecutor extends CommandExecutor {
+ private String namenode = null;
+ public FSCmdExecutor(String namenode) {
+ this.namenode = namenode;
+ }
+ protected void execute(final String cmd) throws Exception{
+ FsShell shell = new FsShell();
+ String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
+ ToolRunner.run(shell, args);
+ }
+ }
+}
Added: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java?rev=771635&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestMRCLI.java Tue May 5 09:16:51 2009
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CommandExecutor;
+import org.apache.hadoop.cli.util.CLITestData.TestCmd;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.tools.MRAdmin;
+import org.apache.hadoop.util.ToolRunner;
+
+public class TestMRCLI extends TestHDFSCLI{
+
+ protected MiniMRCluster mrCluster = null;
+ protected String jobtracker = null;
+ protected MRCmdExecutor cmdExecutor = null;
+
+ public void setUp() throws Exception {
+ super.setUp();
+ JobConf mrConf = new JobConf(conf);
+ mrCluster = new MiniMRCluster(1, dfsCluster.getFileSystem().getUri().toString(), 1,
+ null, null, mrConf);
+ jobtracker = mrCluster.createJobConf().get("mapred.job.tracker", "local");
+ cmdExecutor = new MRCmdExecutor(jobtracker);
+ }
+
+
+ public void tearDown() throws Exception {
+ mrCluster.shutdown();
+ super.tearDown();
+ }
+
+ protected String getTestFile() {
+ return "testMRConf.xml";
+ }
+
+ protected String expandCommand(final String cmd) {
+ String expCmd = cmd;
+ expCmd = expCmd.replaceAll("JOBTRACKER", jobtracker);
+ expCmd = super.expandCommand(cmd);
+ return expCmd;
+ }
+
+ protected Result execute(TestCmd cmd) throws Exception {
+ if(cmd.getType() == TestCmd.CommandType.MRADMIN) {
+ return cmdExecutor.executeCommand(cmd.getCmd());
+ }
+ else throw new Exception("Unknow type of Test command:"+ cmd.getType());
+ }
+
+ public static class MRCmdExecutor extends CommandExecutor {
+ private String jobtracker = null;
+ public MRCmdExecutor(String jobtracker) {
+ this.jobtracker = jobtracker;
+ }
+ protected void execute(final String cmd) throws Exception{
+ MRAdmin mradmin = new MRAdmin();
+ String[] args = getCommandAsArgs(cmd, "JOBTRACKER", jobtracker);
+ ToolRunner.run(mradmin, args);
+ }
+ }
+}