You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2013/04/27 03:43:49 UTC
svn commit: r1476464 - in /hadoop/common/branches/branch-1.2: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/JobInProgress.java
src/mapred/org/apache/hadoop/mapred/JobTracker.java
src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
Author: acmurthy
Date: Sat Apr 27 01:43:49 2013
New Revision: 1476464
URL: http://svn.apache.org/r1476464
Log:
Merge -c 1476463 from branch-1 to branch-1.2 to fix MAPREDUCE-5158. Cleanup job-submission related files on HDFS during JobTracker restart for jobs which set mapreduce.job.restart.recover to false to indicate they don't want to be recovered. Contributed by Mayank Bansal.
Modified:
hadoop/common/branches/branch-1.2/CHANGES.txt
hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
Modified: hadoop/common/branches/branch-1.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/CHANGES.txt?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.2/CHANGES.txt Sat Apr 27 01:43:49 2013
@@ -594,6 +594,11 @@ Release 1.2.0 - 2013.04.16
HADOOP-9502. chmod/chown do not return error exit codes for some exceptions.
(szetszwo)
+ MAPREDUCE-5158. Cleanup job-submission related files on HDFS during
+ JobTracker restart for jobs which set mapreduce.job.restart.recover to
+ false to indicate they don't want to be recovered. (Mayank Bansal via
+ acmurthy)
+
Release 1.1.2 - 2013.01.30
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Sat Apr 27 01:43:49 2013
@@ -3298,6 +3298,16 @@ public class JobInProgress {
jobtracker.storeCompletedJob(this);
jobtracker.finalizeJob(this);
+ }
+ cleanupJob();
+ }
+
+ /**
+ * The job is dead. We're now cleaning it, getting rid of job directories and
+ * removing all delegation token etc.
+ */
+ void cleanupJob() {
+ synchronized (this) {
try {
// Definitely remove the local-disk copy of the job file
if (localJobFile != null) {
Modified: hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/branch-1.2/src/mapred/org/apache/hadoop/mapred/JobTracker.java Sat Apr 27 01:43:49 2013
@@ -3609,6 +3609,8 @@ public class JobTracker implements MRCon
!job.getJobConf().getBoolean(
JobConf.MAPREDUCE_RECOVER_JOB,
JobConf.DEFAULT_MAPREDUCE_RECOVER_JOB)) {
+ LOG.info("Job "+ jobId.toString() + " is not enable for recovery, cleaning up job files");
+ job.cleanupJob();
return null;
}
Modified: hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=1476464&r1=1476463&r2=1476464&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java (original)
+++ hadoop/common/branches/branch-1.2/src/test/org/apache/hadoop/mapred/TestRecoveryManager.java Sat Apr 27 01:43:49 2013
@@ -257,6 +257,8 @@ public class TestRecoveryManager {
UtilsForTests.waitFor(100);
}
Assert.assertTrue("Task should be successful", rJob1.isSuccessful());
+ Assert.assertTrue("Job should be cleaned up", !fs.exists(new Path(job1.get("mapreduce.job.dir"))));
+ Assert.assertTrue("Job should be cleaned up", !fs.exists(new Path(job2.get("mapreduce.job.dir"))));
}
public static class TestJobTrackerInstrumentation extends JobTrackerInstrumentation {