You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 06:57:19 UTC
svn commit: r1079219 - in /hadoop/mapreduce/branches/yahoo-merge/src:
java/org/apache/hadoop/mapred/JobInProgress.java
test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
Author: omalley
Date: Tue Mar 8 05:57:18 2011
New Revision: 1079219
URL: http://svn.apache.org/viewvc?rev=1079219&view=rev
Log:
commit ce9420a251cac5a2e3273818f967680b1dded3cc
Author: Greg Roelofs <ro...@yahoo-inc.com>
Date: Tue Dec 14 19:12:31 2010 -0800
Clarify FIXMEs; remove debugging noise.
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079219&r1=1079218&r2=1079219&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Mar 8 05:57:18 2011
@@ -776,9 +776,10 @@ public class JobInProgress {
return maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded;
}
-//GRR FIXME? who calls this, and what should it return for ubertasks?
+ // called only by JT's initJob() (and test-related clones), which calls it
+ // only after calling initTasks(), which may modify jobSetupCleanupNeeded:
synchronized boolean isSetupCleanupRequired() {
- return jobSetupCleanupNeeded;
+ return jobSetupCleanupNeeded;
}
// Should be called once the init is done. This will complete the job
@@ -891,7 +892,6 @@ public class JobInProgress {
// case), so henceforth we pretend that setup and cleanup aren't needed
// --unless/until job fails or is killed, in which case a separate
// cleanup task will be triggered
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress initSetupCleanupTasks(): forcing jobSetupCleanupNeeded to false");
jobSetupCleanupNeeded = false;
return;
}
@@ -1498,19 +1498,16 @@ System.out.println("GRR DEBUG (" + Strin
int numUniqueHosts,
boolean isMapSlot
) throws IOException {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): starting");
if(!tasksInited.get() || !jobSetupCleanupNeeded) {
return null;
}
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): about to enter locked section");
synchronized(this) {
if (!canLaunchJobCleanupTask()) {
return null;
}
String taskTracker = tts.getTrackerName();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): taskTracker = " + taskTracker);
// Update the last-known clusterSize
this.clusterSize = clusterSize;
if (!shouldRunOnTaskTracker(taskTracker)) {
@@ -1525,14 +1522,12 @@ System.out.println("GRR DEBUG (" + Strin
}
TaskInProgress tip = findTaskFromList(cleanupTaskList,
tts, numUniqueHosts, false);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): found cleanup TIP " + tip);
if (tip == null) {
return null;
}
// Now launch the cleanupTask
Task result = tip.getTaskToRun(tts.getTrackerName());
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobCleanupTask(): launching cleanup Task " + result);
if (result != null) {
addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
if (jobFailed) {
@@ -1594,13 +1589,11 @@ System.out.println("GRR DEBUG (" + Strin
int numUniqueHosts,
boolean isMapSlot
) throws IOException {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): starting");
- if(!tasksInited.get() || !jobSetupCleanupNeeded) {
+ // uberMode condition should be redundant, but make sure anyway:
+ if (!tasksInited.get() || !jobSetupCleanupNeeded || uberMode) {
return null;
}
-//GRR FIXME: need special protection here (since no setup TIPs for ubermode)? may need to split jobSetupCleanupNeeded into setup and cleanup halves...
-
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): about to enter locked section");
+
synchronized(this) {
if (!canLaunchSetupTask()) {
return null;
@@ -1626,7 +1619,6 @@ System.out.println("GRR DEBUG (" + Strin
// Now launch the setupTask
Task result = tip.getTaskToRun(tts.getTrackerName());
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress obtainJobSetupTask(): launching setup Task " + result);
if (result != null) {
addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
}
@@ -3067,7 +3059,6 @@ System.out.println("GRR DEBUG (" + Strin
if (uberMode) {
// restore setup/cleanup status so separate cleanup task will be launched
// (see obtainJobCleanupTask())
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): JobInProgress terminate(): restoring jobSetupCleanupNeeded to " + jobSetupCleanupNeeded);
jobSetupCleanupNeeded = uberSetupCleanupNeeded;
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1079219&r1=1079218&r2=1079219&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java Tue Mar 8 05:57:18 2011
@@ -98,10 +98,8 @@ public class TestJobCleanup extends Test
System.err.println("---- HERE ----");
JobConf conf = context.getJobConf();
Path outputPath = FileOutputFormat.getOutputPath(conf);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup CommitterWithCustomDeprecatedCleanup (FileOutputCommitter) cleanupJob(): creating " + outputPath + "/" + CUSTOM_CLEANUP_FILE_NAME);
FileSystem fs = outputPath.getFileSystem(conf);
fs.create(new Path(outputPath, CUSTOM_CLEANUP_FILE_NAME)).close();
-//try { Thread.sleep(20000); /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException ie) { }
}
}
@@ -118,14 +116,11 @@ System.out.println("GRR DEBUG (" + Strin
String fileName = (state == JobStatus.FAILED)
? TestJobCleanup.ABORT_FAILED_FILE_NAME
: TestJobCleanup.ABORT_KILLED_FILE_NAME;
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup CommitterWithCustomAbort (FileOutputCommitter) abortJob(): creating " + outputPath + "/" + fileName);
fs.create(new Path(outputPath, fileName)).close();
-//try { Thread.sleep(20000); /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException ie) { }
}
}
private Path getNewOutputDir() {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup getNewOutputDir(): creating " + TEST_ROOT_DIR + "/output-" + outDirs);
return new Path(TEST_ROOT_DIR, "output-" + outDirs++);
}
@@ -155,17 +150,14 @@ System.out.println("GRR DEBUG (" + Strin
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): JobID = " + id);
job.waitForCompletion();
Path testFile = new Path(outDir, filename);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): checking for existence of done file: " + testFile);
assertTrue("Done file missing for job " + id, fileSys.exists(testFile));
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testSuccessfulJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for successful job "
+ id, fileSys.exists(file));
}
@@ -186,12 +178,10 @@ System.out.println("GRR DEBUG (" + Strin
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): JobID = " + id);
job.waitForCompletion();
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): checking for existence of done file: " + testFile);
assertTrue("File " + testFile + " missing for failed job " + id,
fileSys.exists(testFile));
}
@@ -199,7 +189,6 @@ System.out.println("GRR DEBUG (" + Strin
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testFailedJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for failed job "
+ id, fileSys.exists(file));
}
@@ -219,7 +208,6 @@ System.out.println("GRR DEBUG (" + Strin
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.submitJob(jc);
JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): JobID = " + id);
JobInProgress jip =
mr.getJobTrackerRunner().getJobTracker().getJob(job.getID());
@@ -238,7 +226,6 @@ System.out.println("GRR DEBUG (" + Strin
if (fileName != null) {
Path testFile = new Path(outDir, fileName);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): checking for existence of done file: " + testFile);
assertTrue("File " + testFile + " missing for job " + id,
fileSys.exists(testFile));
}
@@ -246,7 +233,6 @@ System.out.println("GRR DEBUG (" + Strin
// check if the files from the missing set exists
for (String ex : exclude) {
Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testKilledJob(): checking for nonexistence of 'missing' file: " + file);
assertFalse("File " + file + " should not be present for killed job "
+ id, fileSys.exists(file));
}
@@ -281,20 +267,17 @@ System.out.println("GRR DEBUG (" + Strin
*/
public void testCustomAbort() throws IOException {
// check with a successful job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testSuccessfulJob(" + FileOutputCommitter.SUCCEEDED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testSuccessfulJob(FileOutputCommitter.SUCCEEDED_FILE_NAME,
CommitterWithCustomAbort.class,
new String[] {ABORT_FAILED_FILE_NAME,
ABORT_KILLED_FILE_NAME});
// check with a failed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testFailedJob(" + ABORT_FAILED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testFailedJob(ABORT_FAILED_FILE_NAME, CommitterWithCustomAbort.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
ABORT_KILLED_FILE_NAME});
// check with a killed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomAbort(): calling testKilledJob(" + ABORT_KILLED_FILE_NAME + ", CommitterWithCustomAbort.class, ...)");
testKilledJob(ABORT_KILLED_FILE_NAME, CommitterWithCustomAbort.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
ABORT_FAILED_FILE_NAME});
@@ -307,19 +290,16 @@ System.out.println("GRR DEBUG (" + Strin
*/
public void testCustomCleanup() throws IOException {
// check with a successful job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testSuccessfulJob(" + CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testSuccessfulJob(CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {});
// check with a failed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testFailedJob(" + CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testFailedJob(CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});
// check with a killed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis()) + "): TestJobCleanup testCustomCleanup(): calling testKilledJob(" + TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME + ", CommitterWithCustomDeprecatedCleanup.class, ...)");
testKilledJob(TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME,
CommitterWithCustomDeprecatedCleanup.class,
new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});