You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 06:57:13 UTC
svn commit: r1079218 - in /hadoop/mapreduce/branches/yahoo-merge/src:
contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
java/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapred/
webapps/job/
Author: omalley
Date: Tue Mar 8 05:57:12 2011
New Revision: 1079218
URL: http://svn.apache.org/viewvc?rev=1079218&view=rev
Log:
commit 839669d69fbc0528ac1855611c659214a5bfd19c
Author: Greg Roelofs <ro...@yahoo-inc.com>
Date: Tue Dec 14 18:44:39 2010 -0800
Added fix for failed/killed-uberjob cleanup; removed redundant JIP accessors
for speculation; explicitly forced speculation off for uber mode; fixed four
unit tests; renamed JIP getUberMode() accessor to more expected isUber();
fixed order of expected/actual values in TestBadRecords asserts. (Still
lots of debug junk--will remove in next commit.)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp
Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Tue Mar 8 05:57:12 2011
@@ -630,9 +630,8 @@ class CapacityTaskScheduler extends Task
boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
//Check if job supports speculative map execution first then
//check if job has speculative maps.
- return (job.getMapSpeculativeExecution()) && (
- hasSpeculativeTask(job.getTasks(TaskType.MAP),
- tts));
+ return (job.hasSpeculativeMaps() &&
+ hasSpeculativeTask(job.getTasks(TaskType.MAP), tts));
}
}
@@ -677,9 +676,8 @@ class CapacityTaskScheduler extends Task
boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
//check if the job supports reduce speculative execution first then
//check if the job has speculative tasks.
- return (job.getReduceSpeculativeExecution()) && (
- hasSpeculativeTask(job.getTasks(TaskType.REDUCE),
- tts));
+ return (job.hasSpeculativeReduces() &&
+ hasSpeculativeTask(job.getTasks(TaskType.REDUCE), tts));
}
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Mar 8 05:57:12 2011
@@ -141,6 +141,8 @@ public class JobInProgress {
// uber internals tracked here to avoid need for new TIP get-accessors
private int uberMapTasks = 0;
private int uberReduceTasks = 0;
+ private boolean uberSetupCleanupNeeded = false;
+ private boolean uberMode = false;
int mapFailuresPercent = 0;
int reduceFailuresPercent = 0;
@@ -151,7 +153,6 @@ public class JobInProgress {
private volatile boolean jobKilled = false;
private volatile boolean jobFailed = false;
private /* final */ boolean jobSetupCleanupNeeded;
- private boolean uberMode = false;
JobPriority priority = JobPriority.NORMAL;
protected JobTracker jobtracker;
@@ -575,18 +576,10 @@ public class JobInProgress {
/**
* Was the job small enough to uberize?
*/
- public boolean getUberMode() {
+ public boolean isUber() {
return uberMode;
}
- boolean getMapSpeculativeExecution() {
- return hasSpeculativeMaps;
- }
-
- boolean getReduceSpeculativeExecution() {
- return hasSpeculativeReduces;
- }
-
long getMemoryForMapTask() {
return memoryPerMap;
}
@@ -696,14 +689,19 @@ public class JobInProgress {
|| sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT);
if (uberMode) {
- // save internal details for UI
+ // save internal details for UI and abort-cleanup
uberMapTasks = numMapTasks;
uberReduceTasks = numReduceTasks;
+ uberSetupCleanupNeeded = jobSetupCleanupNeeded;
- // this method modifies numMapTasks (-> 0), numReduceTasks (-> 1), and
- // jobSetupCleanupNeeded (-> false) [and actually creates a TIP, not a
- // true Task; latter is created in TaskInProgress's addRunningTask()
- // method]
+ // disable speculation: makes no sense to speculate an entire job
+ hasSpeculativeMaps = hasSpeculativeReduces = false;
+
+ // This method modifies numMapTasks (-> 0) and numReduceTasks (-> 1)
+ // [and actually creates a TIP, not a true Task; latter is created in
+ // TaskInProgress's addRunningTask() method after JT/scheduler logic
+ // triggers a LaunchTaskAction]. jobSetupCleanupNeeded is left alone
+ // until initSetupCleanupTasks() below.
createUberTask(jobFile.toString(), taskSplitMetaInfo);
}
@@ -739,12 +737,10 @@ public class JobInProgress {
// Creates setup[] and cleanup[] arrays of TIPs, with TaskID indices above
// regular map and reduce values. In uber mode, UberTask itself handles
- // setup and cleanup duties, and jobSetupCleanupNeeded is forced false.
- // But we skip initSetupCleanupTasks() call in that case anyway to avoid
- // a potentially misleading log msg.
- if (!uberMode) {
- initSetupCleanupTasks(jobFile.toString());
- }
+ // setup and cleanup duties, and jobSetupCleanupNeeded is forced false,
+ // but only after creating cleanup TIPs (in case job fails or is killed =>
+ // still want abortTask() cleanup [possibly custom] to happen).
+ initSetupCleanupTasks(jobFile.toString());
synchronized(jobInitKillStatus){
jobInitKillStatus.initDone = true;
@@ -780,6 +776,7 @@ public class JobInProgress {
return maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded;
}
+//GRR FIXME? who calls this, and what should it return for ubertasks?
synchronized boolean isSetupCleanupRequired() {
return jobSetupCleanupNeeded;
}
@@ -864,7 +861,6 @@ public class JobInProgress {
nonRunningReduces.add(reduces[0]); // note: no map analogue
numMapTasks = 0;
numReduceTasks = 1;
- jobSetupCleanupNeeded = false;
LOG.info("Input size for job " + jobId + " = " + inputLength
+ ". Number of splits = " + splits.length + ". UberTasking!");
}
@@ -890,6 +886,16 @@ public class JobInProgress {
numReduceTasks, jobtracker, conf, this, 1);
cleanup[1].setJobCleanupTask();
+ if (uberMode) {
+ // ubertasks handle setup internally (as well as cleanup in the normal
+ // case), so henceforth we pretend that setup and cleanup aren't needed
+ // --unless/until job fails or is killed, in which case a separate
+ // cleanup task will be triggered
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress initSetupCleanupTasks(): forcing jobSetupCleanupNeeded to false");
+ jobSetupCleanupNeeded = false;
+ return;
+ }
+
// Create two setup tips, one map and one reduce. Only one will be used.
setup = new TaskInProgress[2];
@@ -1492,16 +1498,19 @@ public class JobInProgress {
int numUniqueHosts,
boolean isMapSlot
) throws IOException {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): starting");
if(!tasksInited.get() || !jobSetupCleanupNeeded) {
return null;
}
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): about to enter locked section");
synchronized(this) {
if (!canLaunchJobCleanupTask()) {
return null;
}
String taskTracker = tts.getTrackerName();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): taskTracker = " + taskTracker);
// Update the last-known clusterSize
this.clusterSize = clusterSize;
if (!shouldRunOnTaskTracker(taskTracker)) {
@@ -1516,12 +1525,14 @@ public class JobInProgress {
}
TaskInProgress tip = findTaskFromList(cleanupTaskList,
tts, numUniqueHosts, false);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): found cleanup TIP " + tip);
if (tip == null) {
return null;
}
// Now launch the cleanupTask
Task result = tip.getTaskToRun(tts.getTrackerName());
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): launching cleanup Task " + result);
if (result != null) {
addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
if (jobFailed) {
@@ -1583,10 +1594,13 @@ public class JobInProgress {
int numUniqueHosts,
boolean isMapSlot
) throws IOException {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): starting");
if(!tasksInited.get() || !jobSetupCleanupNeeded) {
return null;
}
+//GRR FIXME: need special protection here (since no setup TIPs for ubermode)? may need to split jobSetupCleanupNeeded into setup and cleanup halves...
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): about to enter locked section");
synchronized(this) {
if (!canLaunchSetupTask()) {
return null;
@@ -1612,6 +1626,7 @@ public class JobInProgress {
// Now launch the setupTask
Task result = tip.getTaskToRun(tts.getTrackerName());
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): launching setup Task " + result);
if (result != null) {
addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
}
@@ -2239,7 +2254,7 @@ public class JobInProgress {
}
}
- return slowestTIP;
+ return slowestTIP;
}
/**
@@ -3043,12 +3058,19 @@ public class JobInProgress {
* @param jobTerminationState job termination state
*/
private synchronized void terminate(int jobTerminationState) {
- if(!tasksInited.get()) {
- //init could not be done, we just terminate directly.
+ if (!tasksInited.get()) {
+ // init could not be done, we just terminate directly.
terminateJob(jobTerminationState);
return;
}
+ if (uberMode) {
+ // restore setup/cleanup status so separate cleanup task will be launched
+ // (see obtainJobCleanupTask())
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress terminate(): restoring jobSetupCleanupNeeded to " + jobSetupCleanupNeeded);
+ jobSetupCleanupNeeded = uberSetupCleanupNeeded;
+ }
+
if ((status.getRunState() == JobStatus.RUNNING) ||
(status.getRunState() == JobStatus.PREP)) {
LOG.info("Killing job '" + this.status.getJobID() + "'");
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java Tue Mar 8 05:57:12 2011
@@ -73,6 +73,7 @@ public class TestBadRecords extends Clus
conf.setNumMapTasks(1);
conf.setNumReduceTasks(1);
conf.setInt(JobContext.TASK_TIMEOUT, 30*1000);
+ conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
SkipBadRecords.setMapperMaxSkipRecords(conf, Long.MAX_VALUE);
SkipBadRecords.setReducerMaxSkipGroups(conf, Long.MAX_VALUE);
@@ -112,28 +113,28 @@ public class TestBadRecords extends Clus
LOG.info(runningJob.getCounters().toString());
assertTrue(runningJob.isSuccessful());
- //validate counters
+ // validate counters (1st assert arg is _expected_ value!)
Counters counters = runningJob.getCounters();
- assertEquals(counters.findCounter(TaskCounter.MAP_SKIPPED_RECORDS).
- getCounter(),mapperBadRecords.size());
+ assertEquals(mapperBadRecords.size(),
+ counters.findCounter(TaskCounter.MAP_SKIPPED_RECORDS).getCounter());
int mapRecs = input.size() - mapperBadRecords.size();
- assertEquals(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).
- getCounter(),mapRecs);
- assertEquals(counters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).
- getCounter(),mapRecs);
+ assertEquals(mapRecs,
+ counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getCounter());
+ assertEquals(mapRecs,
+ counters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getCounter());
int redRecs = mapRecs - redBadRecords.size();
- assertEquals(counters.findCounter(TaskCounter.REDUCE_SKIPPED_RECORDS).
- getCounter(),redBadRecords.size());
- assertEquals(counters.findCounter(TaskCounter.REDUCE_SKIPPED_GROUPS).
- getCounter(),redBadRecords.size());
- assertEquals(counters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).
- getCounter(),redRecs);
- assertEquals(counters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).
- getCounter(),redRecs);
- assertEquals(counters.findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS).
- getCounter(),redRecs);
+ assertEquals(redBadRecords.size(),
+ counters.findCounter(TaskCounter.REDUCE_SKIPPED_RECORDS).getCounter());
+ assertEquals(redBadRecords.size(),
+ counters.findCounter(TaskCounter.REDUCE_SKIPPED_GROUPS).getCounter());
+ assertEquals(redRecs,
+ counters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getCounter());
+ assertEquals(redRecs,
+ counters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getCounter());
+ assertEquals(redRecs,
+ counters.findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS).getCounter());
//validate skipped records
Path skipDir = SkipBadRecords.getSkipOutputPath(conf);
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java Tue Mar 8 05:57:12 2011
@@ -151,8 +151,8 @@ public class TestDebugScript {
File output = TaskLog.getRealTaskLogFileLocation(taskId, false,
TaskLog.LogName.DEBUGOUT);
// Check the presence of the output file if the script is to be run.
- assertTrue("Output file does not exists. DebugScript has not been run",
- output.exists());
+ assertTrue("Output file does not exist. DebugScript has not been run.",
+ output != null && output.exists());
// slurp the output from file, which is one line
BufferedReader reader = new BufferedReader(new FileReader(output));
String out = reader.readLine();
@@ -183,6 +183,7 @@ public class TestDebugScript {
throws IOException {
conf.setMapDebugScript(SCRIPT_FILE);
conf.setMaxMapAttempts(0);
+ conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
conf.set("mapred.committer.job.setup.cleanup.needed", "false");
RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir);
return rJob.getID();
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java Tue Mar 8 05:57:12 2011
@@ -131,6 +131,7 @@ public class TestIsolationRunner extends
conf.setKeepTaskFilesPattern(".*");
conf.set(SIDE_EFFECT_DIR_PROPERTY, TEST_ROOT_DIR +
"/isolationrunnerjob/sideeffect");
+ conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
// Delete previous runs' data.
deleteSideEffectFiles(conf);
JobID jobId = runJobNormally(conf);
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java Tue Mar 8 05:57:12 2011
@@ -98,8 +98,10 @@ public class TestJobCleanup extends Test
System.err.println("---- HERE ----");
JobConf conf = context.getJobConf();
Path outputPath = FileOutputFormat.getOutputPath(conf);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup CommitterWithCustomDeprecatedCleanup (FileOutputCommitter) cleanupJob(): creating " + outputPath + "/" + CUSTOM_CLEANUP_FILE_NAME);
FileSystem fs = outputPath.getFileSystem(conf);
fs.create(new Path(outputPath, CUSTOM_CLEANUP_FILE_NAME)).close();
+//try { Thread.sleep(20000); /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException ie) { }
}
}
@@ -116,11 +118,14 @@ public class TestJobCleanup extends Test
String fileName = (state == JobStatus.FAILED)
? TestJobCleanup.ABORT_FAILED_FILE_NAME
: TestJobCleanup.ABORT_KILLED_FILE_NAME;
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup CommitterWithCustomAbort (FileOutputCommitter) abortJob(): creating " + outputPath + "/" + fileName);
fs.create(new Path(outputPath, fileName)).close();
+//try { Thread.sleep(20000); /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException ie) { }
}
}
private Path getNewOutputDir() {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup getNewOutputDir(): creating " + TEST_ROOT_DIR + "/output-" + outDirs);
return new Path(TEST_ROOT_DIR, "output-" + outDirs++);
}
@@ -150,14 +155,17 @@ public class TestJobCleanup extends Test
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): JobID = " + id);
job.waitForCompletion();
Path testFile = new Path(outDir, filename);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): checking for existence of done file: " + testFile);
assertTrue("Done file missing for job " + id, fileSys.exists(testFile));
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for successful job "
+ id, fileSys.exists(file));
}
@@ -178,10 +186,12 @@ public class TestJobCleanup extends Test
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): JobID = " + id);
job.waitForCompletion();
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): checking for existence of done file: " + testFile);
assertTrue("File " + testFile + " missing for failed job " + id,
fileSys.exists(testFile));
}
@@ -189,6 +199,7 @@ public class TestJobCleanup extends Test
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for failed job "
+ id, fileSys.exists(file));
}
@@ -208,12 +219,14 @@ public class TestJobCleanup extends Test
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): JobID = " + id);
JobInProgress jip =
mr.getJobTrackerRunner().getJobTracker().getJob(job.getID());
// wait for the map to be launched
while (true) {
- if (jip.runningMaps() == 1) {
+ if (jip.runningMaps() == 1
+ || (jip.isUber() && jip.runningReduces() == 1)) {
break;
}
UtilsForTests.waitFor(100);
@@ -225,6 +238,7 @@ public class TestJobCleanup extends Test
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): checking for existence of done file: " + testFile);
assertTrue("File " + testFile + " missing for job " + id,
fileSys.exists(testFile));
}
@@ -232,6 +246,7 @@ public class TestJobCleanup extends Test
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for killed job "
+ id, fileSys.exists(file));
}
@@ -266,17 +281,20 @@ public class TestJobCleanup extends Test
*/
public void testCustomAbort() throws IOException {
// check with a successful job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testSuccessfulJob(" + FileOutputCommitter.SUCCEEDED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testSuccessfulJob(FileOutputCommitter.SUCCEEDED_FILE_NAME,
CommitterWithCustomAbort.class,
new String[] {ABORT_FAILED_FILE_NAME,
ABORT_KILLED_FILE_NAME});
// check with a failed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testFailedJob(" + ABORT_FAILED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testFailedJob(ABORT_FAILED_FILE_NAME, CommitterWithCustomAbort.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
ABORT_KILLED_FILE_NAME});
// check with a killed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testKilledJob(" + ABORT_KILLED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testKilledJob(ABORT_KILLED_FILE_NAME, CommitterWithCustomAbort.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
ABORT_FAILED_FILE_NAME});
@@ -289,16 +307,19 @@ public class TestJobCleanup extends Test
*/
public void testCustomCleanup() throws IOException {
// check with a successful job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testSuccessfulJob(" + CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testSuccessfulJob(CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {});
// check with a failed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testFailedJob(" + CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testFailedJob(CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});
// check with a killed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testKilledJob(" + TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testKilledJob(TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});
Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp Tue Mar 8 05:57:12 2011
@@ -365,7 +365,7 @@
"<th>Killed</th>" +
"<th><a href=\"jobfailures.jsp?jobid=" + jobId +
"\">Failed/Killed<br>Task Attempts</a></th></tr>\n");
- if (job.getUberMode()) {
+ if (job.isUber()) {
/* placeholder until true task- and attempt-level uber info available */
printTaskSummary(out, jobId, "uber", status.reduceProgress(),
job.getTasks(TaskType.REDUCE));
@@ -452,7 +452,7 @@ if("off".equals(session.getAttribute("ma
<%} }%>
<%if (job.getTasks(TaskType.REDUCE).length > 0) { %>
-<%if (job.getUberMode()) { %>
+<%if (job.isUber()) { %>
<hr>UberTask Completion Graph -
<%} else { %>
<hr>Reduce Completion Graph -