You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/01/20 01:15:02 UTC
svn commit: r1061071 - in /hadoop/mapreduce/branches/branch-0.22: CHANGES.txt
src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
Author: todd
Date: Thu Jan 20 00:15:01 2011
New Revision: 1061071
URL: http://svn.apache.org/viewvc?rev=1061071&view=rev
Log:
MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test. Contributed by Todd Lipcon
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1061071&r1=1061070&r2=1061071&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Thu Jan 20 00:15:01 2011
@@ -178,6 +178,8 @@ Release 0.22.0 - Unreleased
MAPREDUCE-1878. Add MRUnit documentation. (Aaron Kimball via tomwhite)
+ MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test (todd)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java?rev=1061071&r1=1061070&r2=1061071&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java Thu Jan 20 00:15:01 2011
@@ -21,23 +21,63 @@ import org.apache.hadoop.mapreduce.serve
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.conf.Configuration;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.HttpURLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
import static org.junit.Assert.*;
/**
* System tests for the fair scheduler. These run slower than the
* mock-based tests in TestFairScheduler but have a better chance
* of catching synchronization bugs with the real JT.
+ *
+ * This test suite will often be run inside JCarder in order to catch
+ * deadlock bugs which have plagued the scheduler in the past - hence
+ * it is a bit of a "grab-bag" of system tests, since it's important
+ * that they all run as part of the same JVM instantiation.
*/
public class TestFairSchedulerSystem {
- static final int NUM_THREADS=5;
+ static final int NUM_THREADS=2;
+
+ static MiniMRCluster mr;
+ static JobConf conf;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ conf = new JobConf();
+ final int taskTrackers = 1;
+
+ // Bump up the frequency of preemption updates to test against
+ // deadlocks, etc.
+ conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
+ conf.set("mapred.fairscheduler.update.interval", "1");
+ conf.set("mapred.fairscheduler.preemption.interval", "1");
+ conf.set("mapred.fairscheduler.preemption", "true");
+ conf.set("mapred.fairscheduler.eventlog.enabled", "true");
+ conf.set(JTConfig.JT_PERSIST_JOBSTATUS, "false");
+ mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, conf);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (mr != null) {
+ mr.shutdown();
+ }
+ }
private void runSleepJob(JobConf conf) throws Exception {
String[] args = { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
@@ -45,44 +85,64 @@ public class TestFairSchedulerSystem {
}
/**
- * Bump up the frequency of preemption updates to test against
- * deadlocks, etc.
+ * Submit some concurrent sleep jobs, and visit the scheduler servlet
+ * while they're running.
*/
@Test
- public void testPreemptionUpdates() throws Exception {
- MiniMRCluster mr = null;
- try {
- final int taskTrackers = 1;
-
- JobConf conf = new JobConf();
- conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
- conf.set("mapred.fairscheduler.update.interval", "0");
- conf.set("mapred.fairscheduler.preemption.interval", "0");
- conf.set("mapred.fairscheduler.preemption", "true");
- conf.set("mapred.fairscheduler.eventlog.enabled", "true");
- conf.set(JTConfig.JT_PERSIST_JOBSTATUS, "false");
-
- mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, conf);
- final MiniMRCluster finalMR = mr;
- ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
- List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
- for (int i = 0; i < NUM_THREADS; i++) {
- futures.add(exec.submit(new Callable<Void>() {
- public Void call() throws Exception {
- JobConf jobConf = finalMR.createJobConf();
- runSleepJob(jobConf);
- return null;
- }
- }));
- }
+ public void testFairSchedulerSystem() throws Exception {
+ ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
+ List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
+ for (int i = 0; i < NUM_THREADS; i++) {
+ futures.add(exec.submit(new Callable<Void>() {
+ public Void call() throws Exception {
+ JobConf jobConf = mr.createJobConf();
+ runSleepJob(jobConf);
+ return null;
+ }
+ }));
+ }
- for (Future<Void> future : futures) {
- future.get();
- }
- } finally {
- if (mr != null) { mr.shutdown();
+ // Wait for the tasks to finish, and visit the scheduler servlet
+ // every few seconds while waiting.
+ for (Future<Void> future : futures) {
+ while (true) {
+ try {
+ future.get(3, TimeUnit.SECONDS);
+ break;
+ } catch (TimeoutException te) {
+ // It's OK
+ }
+ checkServlet(true);
+ checkServlet(false);
}
}
-
}
+
+ /**
+ * Check the fair scheduler servlet for good status code and smoke test
+ * for contents.
+ */
+ private void checkServlet(boolean advanced) throws Exception {
+ String jtURL = "http://localhost:" +
+ mr.getJobTrackerRunner().getJobTrackerInfoPort();
+ URL url = new URL(jtURL + "/scheduler" +
+ (advanced ? "?advanced" : ""));
+ HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ assertEquals(200, connection.getResponseCode());
+
+ // Just to be sure, slurp the content and make sure it looks like the scheduler
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(connection.getInputStream()));
+ StringBuilder sb = new StringBuilder();
+
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line).append('\n');
+ }
+
+ String contents = sb.toString();
+ assertTrue(contents.contains("Fair Scheduler Administration"));
+ }
}