You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/04/26 22:50:33 UTC
svn commit: r532855 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobTracker.java
Author: cutting
Date: Thu Apr 26 13:50:32 2007
New Revision: 532855
URL: http://svn.apache.org/viewvc?view=rev&rev=532855
Log:
HADOOP-1279. Fix JobTracker to maintain list of recently completed jobs by order of completion, not submission. Contributed by Arun.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=532855&r1=532854&r2=532855
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Apr 26 13:50:32 2007
@@ -259,6 +259,10 @@
of the checksum file when a checksum error is detected.
(Hairong Kuang via tomwhite)
+78. HADOOP-1279. Fix JobTracker to maintain list of recently
+ completed jobs by order of completion, not submission.
+ (Arun C Murthy via cutting)
+
Release 0.12.3 - 2007-04-06
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?view=diff&rev=532855&r1=532854&r2=532855
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Apr 26 13:50:32 2007
@@ -353,15 +353,20 @@
jobs.remove(job.getProfile().getJobId());
jobInitQueue.remove(job);
jobsByArrival.remove(job);
+ String jobUser = job.getProfile().getUser();
synchronized (userToJobsMap) {
ArrayList<JobInProgress> userJobs =
- userToJobsMap.get(job.getProfile().getUser());
+ userToJobsMap.get(jobUser);
synchronized (userJobs) {
userJobs.remove(job);
}
+ if (userJobs.isEmpty()) {
+ userToJobsMap.remove(jobUser);
+ }
}
LOG.info("Retired job with id: '" +
- job.getProfile().getJobId() + "'");
+ job.getProfile().getJobId() + "' of user '" +
+ jobUser + "'");
}
}
}
@@ -895,11 +900,19 @@
synchronized (jobs) {
synchronized (jobsByArrival) {
synchronized (jobInitQueue) {
- String jobUser = job.getProfile().getUser();
synchronized (userToJobsMap) {
+ String jobUser = job.getProfile().getUser();
+ if (!userToJobsMap.containsKey(jobUser)) {
+ userToJobsMap.put(jobUser,
+ new ArrayList<JobInProgress>());
+ }
ArrayList<JobInProgress> userJobs =
userToJobsMap.get(jobUser);
synchronized (userJobs) {
+ // Add the currently completed 'job'
+ userJobs.add(job);
+
+ // Check if we need to retire some jobs of this user
while (userJobs.size() >
MAX_COMPLETE_USER_JOBS_IN_MEMORY) {
JobInProgress rjob = userJobs.get(0);
@@ -929,7 +942,8 @@
jobsByArrival.remove(rjob);
LOG.info("Retired job with id: '" +
- rjob.getProfile().getJobId() + "'");
+ rjob.getProfile().getJobId() + "' of user: '" +
+ jobUser + "'");
} else {
// Do not remove jobs that aren't complete.
// Stop here, and let the next pass take
@@ -938,6 +952,9 @@
}
}
}
+ if (userJobs.isEmpty()) {
+ userToJobsMap.remove(jobUser);
+ }
}
}
}
@@ -1424,22 +1441,10 @@
synchronized (jobs) {
synchronized (jobsByArrival) {
synchronized (jobInitQueue) {
- synchronized (userToJobsMap) {
- jobs.put(job.getProfile().getJobId(), job);
- String jobUser = job.getProfile().getUser();
- if (!userToJobsMap.containsKey(jobUser)) {
- userToJobsMap.put(jobUser,
- new ArrayList<JobInProgress>());
- }
- ArrayList<JobInProgress> userJobs =
- userToJobsMap.get(jobUser);
- synchronized (userJobs) {
- userJobs.add(job);
- }
- jobsByArrival.add(job);
- jobInitQueue.add(job);
- jobInitQueue.notifyAll();
- }
+ jobs.put(job.getProfile().getJobId(), job);
+ jobsByArrival.add(job);
+ jobInitQueue.add(job);
+ jobInitQueue.notifyAll();
}
}
}