You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:42:05 UTC
svn commit: r1077667 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/ test/org/apache/hadoop/mapred/
Author: omalley
Date: Fri Mar 4 04:42:04 2011
New Revision: 1077667
URL: http://svn.apache.org/viewvc?rev=1077667&view=rev
Log:
commit 483618d2ee7a448c9a824bd86641e0c079d61647
Author: Arun C Murthy <ac...@apache.org>
Date: Wed Sep 8 18:25:29 2010 -0700
MAPREDUCE-2055. Fix JobTracker to decouple job retirement from copy of job-history file to HDFS and enhance RetiredJobInfo to carry aggregated job-counters to prevent a disk roundtrip on job-completion to fetch counters for the JobClient. Contributed by Krishna Ramachandran.
+++ b/YAHOO-CHANGES.txt
+
+ MAPREDUCE-2055. Fix JobTracker to decouple job retirement from copy of
+ job-history file to HDFS and enhance RetiredJobInfo to carry aggregated
+ job-counters to prevent a disk roundtrip on job-completion to fetch
+ counters for the JobClient. (Krishna Ramachandran via acmurthy)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistoryServer.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestRawHistoryFile.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077667&r1=1077666&r2=1077667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Fri Mar 4 04:42:04 2011
@@ -1195,14 +1195,6 @@ public class JobInProgress {
this.historyFile = file;
}
- boolean isHistoryFileCopied() {
- return historyFileCopied;
- }
-
- synchronized void setHistoryFileCopied() {
- this.historyFileCopied = true;
- }
-
/**
* Returns the job-level counters.
*
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=1077667&r1=1077666&r2=1077667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobTracker.java Fri Mar 4 04:42:04 2011
@@ -263,7 +263,6 @@ public class JobTracker implements MRCon
*/
static final int MIN_TIME_BEFORE_RETIRE = 0;
-
private int nextJobId = 1;
public static final Log LOG = LogFactory.getLog(JobTracker.class);
@@ -516,7 +515,6 @@ public class JobTracker implements MRCon
synchronized void historyFileCopied(JobID jobid, String historyFile) {
JobInProgress job = getJob(jobid);
if (job != null) { //found in main cache
- job.setHistoryFileCopied();
if (historyFile != null) {
job.setHistoryFile(historyFile);
}
@@ -534,9 +532,11 @@ public class JobTracker implements MRCon
final JobStatus status;
final JobProfile profile;
final long finishTime;
+ final Counters counters;
private String historyFile;
- RetireJobInfo(JobStatus status, JobProfile profile, long finishTime,
- String historyFile) {
+ RetireJobInfo(Counters counters, JobStatus status, JobProfile profile,
+ long finishTime, String historyFile) {
+ this.counters = counters;
this.status = status;
this.profile = profile;
this.finishTime = finishTime;
@@ -561,7 +561,7 @@ public class JobTracker implements MRCon
}
synchronized void addToCache(JobInProgress job) {
- RetireJobInfo info = new RetireJobInfo(job.getStatus(),
+ RetireJobInfo info = new RetireJobInfo(job.getCounters(), job.getStatus(),
job.getProfile(), job.getFinishTime(), job.getHistoryFile());
jobRetireInfoQ.add(info);
jobIDStatusMap.put(info.status.getJobID(), info);
@@ -590,10 +590,9 @@ public class JobTracker implements MRCon
}
private boolean minConditionToRetire(JobInProgress job, long now) {
- return job.getStatus().getRunState() != JobStatus.RUNNING &&
- job.getStatus().getRunState() != JobStatus.PREP &&
- (job.getFinishTime() + MIN_TIME_BEFORE_RETIRE < now) &&
- job.isHistoryFileCopied();
+ return job.getStatus().getRunState() != JobStatus.RUNNING
+ && job.getStatus().getRunState() != JobStatus.PREP
+ && (job.getFinishTime() + MIN_TIME_BEFORE_RETIRE < now);
}
/**
* The run method lives for the life of the JobTracker,
@@ -4294,7 +4293,6 @@ public class JobTracker implements MRCon
// on the JobTracker since it isn't a synchronized method
return job.getStatus();
} else {
-
RetireJobInfo info = retireJobs.get(jobid);
if (info != null) {
return info.status;
@@ -4315,7 +4313,12 @@ public class JobTracker implements MRCon
aclsManager.checkAccess(job, callerUGI, Operation.VIEW_JOB_COUNTERS);
return isJobInited(job) ? job.getCounters() : EMPTY_COUNTERS;
- }
+ } else {
+ RetireJobInfo info = retireJobs.get(jobid);
+ if (info != null) {
+ return info.counters;
+ }
+ }
}
return completedJobStatusStore.readCounters(jobid);
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistoryServer.java?rev=1077667&r1=1077666&r2=1077667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistoryServer.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobHistoryServer.java Fri Mar 4 04:42:04 2011
@@ -50,7 +50,7 @@ public class TestJobHistoryServer extend
conf.setLong("mapred.job.tracker.retiredjobs.cache.size", 1);
conf.setLong("mapred.jobtracker.retirejob.interval", 0);
conf.setLong("mapred.jobtracker.retirejob.check", 0);
- conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 0);
+ conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 2);
conf.set(JobHistoryServer.MAPRED_HISTORY_SERVER_HTTP_ADDRESS,
"localhost:0");
@@ -86,7 +86,7 @@ public class TestJobHistoryServer extend
conf.setLong("mapred.job.tracker.retiredjobs.cache.size", 1);
conf.setLong("mapred.jobtracker.retirejob.interval", 0);
conf.setLong("mapred.jobtracker.retirejob.check", 0);
- conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 0);
+ conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 2);
conf.set(JobHistoryServer.MAPRED_HISTORY_SERVER_HTTP_ADDRESS,
"localhost:8090");
conf.setBoolean(JobHistoryServer.MAPRED_HISTORY_SERVER_EMBEDDED, false);
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestRawHistoryFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestRawHistoryFile.java?rev=1077667&r1=1077666&r2=1077667&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestRawHistoryFile.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestRawHistoryFile.java Fri Mar 4 04:42:04 2011
@@ -51,7 +51,7 @@ public class TestRawHistoryFile extends
conf.setLong("mapred.job.tracker.retiredjobs.cache.size", 1);
conf.setLong("mapred.jobtracker.retirejob.interval", 0);
conf.setLong("mapred.jobtracker.retirejob.check", 0);
- conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 0);
+ conf.setLong("mapred.jobtracker.completeuserjobs.maximum", 1);
conf.set("mapreduce.history.server.http.address", "localhost:0");
mrCluster = new MiniMRCluster(1, conf.get("fs.default.name"), 1,