You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2009/02/13 06:38:01 UTC
svn commit: r743996 - in /hadoop/core/trunk: CHANGES.txt
src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
src/test/org/apache/hadoop/mapred/UtilsForTests.java
Author: ddas
Date: Fri Feb 13 05:38:01 2009
New Revision: 743996
URL: http://svn.apache.org/viewvc?rev=743996&view=rev
Log:
HADOOP-4220. Changes the JobTracker restart tests so that they take much less time. Contributed by Amar Kamat.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Feb 13 05:38:01 2009
@@ -101,6 +101,9 @@
HADOOP-1722. Adds support for TypedBytes and RawBytes in Streaming.
(Klaas Bosteels via ddas)
+ HADOOP-4220. Changes the JobTracker restart tests so that they take much
+ less time. (Amar Kamat via ddas)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java Fri Feb 13 05:38:01 2009
@@ -121,20 +121,11 @@
mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover",
false);
- // Wait for a minute before submitting a job
- UtilsForTests.waitFor(60 * 1000);
-
mr.startJobTracker();
- // Signal the tasks
- UtilsForTests.signalTasks(dfs, fileSys, true, getMapSignalFile(shareDir),
- getReduceSignalFile(shareDir));
-
// Wait for the JT to be ready
UtilsForTests.waitForJobTracker(jobClient);
- UtilsForTests.waitTillDone(jobClient);
-
// The submitted job should not exist
assertTrue("Submitted job was detected with recovery disabled",
UtilsForTests.getJobStatus(jobClient, id) == null);
@@ -143,14 +134,13 @@
/** Tests a job on jobtracker with restart-recovery turned on.
* Preparation :
* - Configure a job with
- * - num-maps : 50
+ * - num-maps : 24
* - num-reducers : 1
* - Configure the cluster to run 1 reducer
* - Lower the history file block size and buffer
*
* Wait for the job to complete 50%. Note that all the job is configured to
- * use {@link HalfWaitingMapper} and {@link WaitingReducer}. So job will
- * eventually wait on 50%
+ * use {@link HalfWaitingMapper}. So job will eventually wait on 50%
*
* Make a note of the following things
* - Task completion events
@@ -191,7 +181,7 @@
// Ideally the tracker should SYNC with the new/restarted jobtracker
FileSystem fileSys = dfs.getFileSystem();
- final int numMaps = 50;
+ final int numMaps = 24;
final int numReducers = 1;
@@ -224,7 +214,7 @@
mr.getMapTaskCompletionEventsUpdates(0, id, numMaps)
.getMapTaskCompletionEvents();
if (trackerEvents.length < numMaps / 2) {
- UtilsForTests.waitFor(1000);
+ UtilsForTests.waitFor(100);
} else {
break;
}
@@ -242,9 +232,6 @@
mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover",
true);
- // Wait for a minute before submitting a job
- UtilsForTests.waitFor(60 * 1000);
-
mr.startJobTracker();
// Signal the map tasks
@@ -274,7 +261,7 @@
mr.getMapTaskCompletionEventsUpdates(0, id, 2 * numMaps)
.getMapTaskCompletionEvents();
if (trackerEvents.length < jtEvents.length) {
- UtilsForTests.waitFor(1000);
+ UtilsForTests.waitFor(100);
} else {
break;
}
@@ -429,10 +416,11 @@
// Make sure that jobhistory leads to a proper job restart
// So keep the blocksize and the buffer size small
JobConf jtConf = new JobConf();
- jtConf.set("mapred.jobtracker.job.history.block.size", "1024");
- jtConf.set("mapred.jobtracker.job.history.buffer.size", "1024");
+ jtConf.set("mapred.jobtracker.job.history.block.size", "64");
+ jtConf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
- jtConf.setLong("mapred.tasktracker.expiry.interval", 25 * 1000);
+ jtConf.setLong("mapred.tasktracker.expiry.interval", 5000);
+ jtConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
jtConf.setBoolean("mapred.acls.enabled", true);
// get the user group info
UserGroupInformation ugi = UserGroupInformation.getCurrentUGI();
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java Fri Feb 13 05:38:01 2009
@@ -49,7 +49,7 @@
throws IOException {
FileSystem fileSys = dfs.getFileSystem();
JobConf jobConf = mr.createJobConf();
- int numMaps = 50;
+ int numMaps = 30;
int numReds = 1;
String mapSignalFile = UtilsForTests.getMapSignalFile(shareDir);
String redSignalFile = UtilsForTests.getReduceSignalFile(shareDir);
@@ -87,9 +87,6 @@
// Kill the 2nd tasktracker
mr.stopTaskTracker(1);
- // Wait for a minute before submitting a job
- UtilsForTests.waitFor(60 * 1000);
-
// Restart the jobtracker
mr.startJobTracker();
@@ -104,11 +101,17 @@
UtilsForTests.waitTillDone(jobClient);
- // Check if the tasks on the lost tracker got re-executed
+ // Check if the tracker got lost
assertTrue("Tracker killed while the jobtracker was down did not get lost "
+ "upon restart",
jobClient.getClusterStatus().getTaskTrackers()
< mr.getNumTaskTrackers());
+
+ // Check if the tasks on the lost tracker got killed
+ int failedMaps =
+ mr.getJobTrackerRunner().getJobTracker().getJob(id).failedMapTasks;
+ assertTrue("Tasks that were run on the lost tracker were not killed",
+ failedMaps > 0);
}
public void testRestartWithLostTracker() throws IOException {
@@ -144,10 +147,10 @@
// Make sure that jobhistory leads to a proper job restart
// So keep the blocksize and the buffer size small
JobConf jtConf = new JobConf();
- jtConf.set("mapred.jobtracker.job.history.block.size", "1024");
- jtConf.set("mapred.jobtracker.job.history.buffer.size", "1024");
+ jtConf.set("mapred.jobtracker.job.history.block.size", "512");
+ jtConf.set("mapred.jobtracker.job.history.buffer.size", "512");
jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
- jtConf.setLong("mapred.tasktracker.expiry.interval", 25 * 1000);
+ jtConf.setLong("mapred.tasktracker.expiry.interval", 5000);
jtConf.setInt("mapred.reduce.copy.backoff", 4);
mr = new MiniMRCluster(2, namenode, 1, null, null, jtConf);
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java Fri Feb 13 05:38:01 2009
@@ -254,7 +254,7 @@
}
}
if (shouldWait) {
- waitFor(1000);
+ waitFor(100);
} else {
break;
}