You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:34:38 UTC
svn commit: r1077602 - in
/hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred:
HighRamJobHelper.java TestHealthScriptError.java
TestHiRamJobWithBlackListTT.java TestJobSummary.java
Author: omalley
Date: Fri Mar 4 04:34:38 2011
New Revision: 1077602
URL: http://svn.apache.org/viewvc?rev=1077602&view=rev
Log:
commit 58507691b6c5af531543929118dcce6f7cd8fa62
Author: Balaji Rajagopalan <ba...@yahoo-inc.com>
Date: Tue Jul 27 18:47:43 2010 -0700
MAPREDUCE-1964 from https://issues.apache.org/jira/secure/ManageAttachments.jspa?id=12469985
Added:
hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java
hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java
hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java?rev=1077602&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/HighRamJobHelper.java Fri Mar 4 04:34:38 2011
@@ -0,0 +1,55 @@
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.SleepJob;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapreduce.test.system.JobInfo;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Assert;
+
+/**
+ * This is a helper class that is used by test cases to run a high ram job
+ * the intention behind creatint this class is reuse code.
+ *
+ */
+public class HighRamJobHelper {
+
+ public HighRamJobHelper () {
+
+ }
+
+ /**
+ * The method runs the high ram job
+ * @param conf configuration for unning the job
+ * @param jobClient instance
+ * @param remoteJTClient instance
+ * @return the job id of the high ram job
+ * @throws Exception is thrown when the method fails to run the high ram job
+ */
+ public JobID runHighRamJob (Configuration conf, JobClient jobClient,
+ JTProtocol remoteJTClient,String assertMessage) throws Exception {
+ SleepJob job = new SleepJob();
+ String jobArgs []= {"-D","mapred.cluster.max.map.memory.mb=2048",
+ "-D","mapred.cluster.max.reduce.memory.mb=2048",
+ "-D","mapred.cluster.map.memory.mb=1024",
+ "-D","mapred.cluster.reduce.memory.mb=1024",
+ "-m", "6",
+ "-r", "2",
+ "-mt", "2000",
+ "-rt", "2000",
+ "-recordt","100"};
+ JobConf jobConf = new JobConf(conf);
+ jobConf.setMemoryForMapTask(2048);
+ jobConf.setMemoryForReduceTask(2048);
+ int exitCode = ToolRunner.run(jobConf, job, jobArgs);
+ Assert.assertEquals("Exit Code:", 0, exitCode);
+ UtilsForTests.waitFor(1000);
+ JobID jobId = jobClient.getAllJobs()[0].getJobID();
+ JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
+ Assert.assertEquals(assertMessage,
+ jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
+ return jobId;
+ }
+
+}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java?rev=1077602&r1=1077601&r2=1077602&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHealthScriptError.java Fri Mar 4 04:34:38 2011
@@ -17,7 +17,7 @@ public class TestHealthScriptError {
public static MRCluster cluster;
public static HealthScriptHelper helper;
public static String remotePath;
- public String healthScriptError="healthScriptError";
+ public static String healthScriptError="healthScriptError";
public static String remoteHSPath = "test.system.hdrc.healthscript.path";
static final Log LOG = LogFactory.getLog(TestHealthScriptError.class);
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java?rev=1077602&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestHiRamJobWithBlackListTT.java Fri Mar 4 04:34:38 2011
@@ -0,0 +1,96 @@
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.TTClient;
+import org.apache.hadoop.mapreduce.test.system.MRCluster.Role;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import java.util.List;
+import java.util.ArrayList;
+
+public class TestHiRamJobWithBlackListTT {
+ static final Log LOG = LogFactory.getLog(TestHiRamJobWithBlackListTT.class);
+ private static HealthScriptHelper bListHelper = null;
+ public static String remotePath;
+ public static MRCluster cluster;
+
+ @BeforeClass
+ public static void setUp() throws java.lang.Exception {
+ String [] expExcludeList = new String[2];
+ expExcludeList[0] = "java.net.ConnectException";
+ expExcludeList[1] = "java.io.IOException";
+ cluster = MRCluster.createCluster(new Configuration());
+ cluster.setExcludeExpList(expExcludeList);
+ cluster.setUp();
+ bListHelper = new HealthScriptHelper();
+ remotePath = cluster.getConf().get(TestHealthScriptError.remoteHSPath);
+ }
+
+ /** Black List more than 25 % of task trackers , run the high ram
+ * job and make sure that no exception is thrown.
+ * @throws Exception If fails to blacklist TT or run high ram high
+ */
+ @Test
+ public void testHiRamJobBlackListedTaskTrackers() throws Exception {
+ final HighRamJobHelper hRamHelper = new HighRamJobHelper();
+ List<TTClient> bListedTT = new ArrayList<TTClient>();
+ List<TTClient> tClient = cluster.getTTClients();
+ int count = tClient.size();
+ int moreThan25Per = count / 4 +1;
+ LOG.info ("More than 25 % of TTclient is "+moreThan25Per);
+ for (int i=0; i < moreThan25Per ; ++i) {
+ TTClient client = tClient.get(i);
+ bListedTT.add(client);
+ blackListTT(client);
+ }
+ //Now run the high ram job
+ JobClient jobClient = cluster.getJTClient().getClient();
+ JTProtocol remoteJTClient = cluster.getJTClient().getProxy();
+ Configuration conf = remoteJTClient.getDaemonConf();
+ hRamHelper.runHighRamJob(conf, jobClient, remoteJTClient,
+ "Job did not succeed");
+ //put the task tracker back in healthy state
+ for( int i =0; i < bListedTT.size() ; ++i) {
+ unBlackListTT(bListedTT.get(i));
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ cluster.tearDown();
+ }
+
+ private void unBlackListTT (TTClient client) throws Exception{
+ //Now put back the task tracker in a healthy state
+ cluster.restart(client, Role.TT);
+ bListHelper.deleteFileOnRemoteHost(remotePath + File.separator +
+ TestHealthScriptError.healthScriptError, client.getHostName());
+ }
+
+ private void blackListTT(TTClient client) throws Exception {
+ Configuration tConf= client.getProxy().getDaemonConf();
+ tConf.set("mapred.task.tracker.report.address",
+ cluster.getConf().get("mapred.task.tracker.report.address"));
+ String defaultHealthScript = tConf.get("mapred.healthChecker.script.path");
+ Assert.assertTrue("Health script was not set", defaultHealthScript != null);
+ tConf.set("mapred.healthChecker.script.path", remotePath+File.separator+
+ TestHealthScriptError.healthScriptError);
+ tConf.setInt("mapred.healthChecker.interval", 1000);
+ bListHelper.copyFileToRemoteHost(TestHealthScriptError.healthScriptError,
+ client.getHostName(), remotePath, cluster);
+ cluster.restartDaemonWithNewConfig(client, "mapred-site.xml", tConf,
+ Role.TT);
+ //make sure the TT is blacklisted
+ bListHelper.verifyTTBlackList(tConf, client,
+ "ERROR Task Tracker status is fatal", cluster);
+ }
+
+}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java?rev=1077602&r1=1077601&r2=1077602&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/system/java/org/apache/hadoop/mapred/TestJobSummary.java Fri Mar 4 04:34:38 2011
@@ -60,6 +60,7 @@ public class TestJobSummary {
private static Path inputDir = new Path("input");
private static Path outputDir = new Path("output");
private static String queueName = null;
+
@BeforeClass
public static void before() throws Exception {
String [] expExcludeList = {"java.net.ConnectException",
@@ -176,26 +177,10 @@ public class TestJobSummary {
@Test
public void testJobSummaryInfoOfHighMemoryJob() throws IOException,
Exception {
- SleepJob job = new SleepJob();
- String jobArgs []= {"-D","mapred.cluster.max.map.memory.mb=2048",
- "-D","mapred.cluster.max.reduce.memory.mb=2048",
- "-D","mapred.cluster.map.memory.mb=1024",
- "-D","mapred.cluster.reduce.memory.mb=1024",
- "-m", "6",
- "-r", "2",
- "-mt", "2000",
- "-rt", "2000",
- "-recordt","100"};
- JobConf jobConf = new JobConf(conf);
- jobConf.setMemoryForMapTask(2048);
- jobConf.setMemoryForReduceTask(2048);
- int exitCode = runTool(jobConf,job,jobArgs);
- Assert.assertEquals("Exit Code:", 0, exitCode);
- UtilsForTests.waitFor(1000);
- JobID jobId = jobClient.getAllJobs()[0].getJobID();
+ final HighRamJobHelper helper = new HighRamJobHelper();
+ JobID jobId = helper.runHighRamJob(conf, jobClient, remoteJTClient,
+ "Job did not succeed");
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
- Assert.assertEquals("Job has not been succeeded",
- jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
verifyJobSummaryInfo(jInfo,jobId);
}
@@ -239,11 +224,7 @@ public class TestJobSummary {
verifyJobSummaryInfo(jInfo,jobId);
}
- private int runTool(Configuration job, Tool tool,
- String[] jobArgs) throws Exception {
- int returnStatus = ToolRunner.run(job, tool, jobArgs);
- return returnStatus;
- }
+
private void verifyJobSummaryInfo(JobInfo jInfo, JobID id)
throws IOException {
java.util.HashMap<String,String> map = jtClient.getJobSummary(id);