You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sc...@apache.org on 2011/02/03 21:01:09 UTC
svn commit: r1066931 - in /hadoop/mapreduce/branches/branch-0.22:
CHANGES.txt src/java/org/apache/hadoop/mapred/JobInProgress.java
src/java/org/apache/hadoop/mapred/JobTracker.java
src/java/org/apache/hadoop/mapred/TaskInProgress.java
Author: schen
Date: Thu Feb 3 20:01:09 2011
New Revision: 1066931
URL: http://svn.apache.org/viewvc?rev=1066931&view=rev
Log:
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
JobInProgress.getCounters() aquire locks in a shorter time period.
(Joydeep Sen Sarma via schen)
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskInProgress.java
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1066931&r1=1066930&r2=1066931&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Thu Feb 3 20:01:09 2011
@@ -185,13 +185,17 @@ Release 0.22.0 - Unreleased
MAPREDUCE-1783. Delay task Initialization till a job can be run
OPTIMIZATIONS
-
+
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
scalability. (Arun C. Murthy & Richard King via acmurthy)
MAPREDUCE-1829. JobInProgress.findSpeculativeTask should use min() to
find the candidate instead of sort(). (Scott Chen via vinodkv)
+ MAPREDUCE-2026. Make JobTracker.getJobCounters() and
+ JobInProgress.getCounters() aquire locks in a shorter time period.
+ (Joydeep Sen Sarma via schen)
+
BUG FIXES
MAPREDUCE-1845. FairScheduler.tasksToPreempt() can return negative number.
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1066931&r1=1066930&r2=1066931&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobInProgress.java Thu Feb 3 20:01:09 2011
@@ -1253,9 +1253,14 @@ public class JobInProgress {
* Returns the total job counters, by adding together the job,
* the map and the reduce counters.
*/
- public synchronized Counters getCounters() {
+ public Counters getCounters() {
Counters result = new Counters();
- result.incrAllCounters(getJobCounters());
+ synchronized (this) {
+ result.incrAllCounters(getJobCounters());
+ }
+
+ // the counters of TIPs are not updated in place.
+ // hence read-only access is ok without any locks
incrementTaskCounters(result, maps);
return incrementTaskCounters(result, reduces);
}
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1066931&r1=1066930&r2=1066931&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Feb 3 20:01:09 2011
@@ -3505,26 +3505,27 @@ public class JobTracker implements MRCon
throws AccessControlException, IOException {
JobID oldJobID = JobID.downgrade(jobid);
-
+ JobInProgress job;
synchronized (this) {
- JobInProgress job = jobs.get(oldJobID);
- if (job != null) {
- // check the job-access
- aclsManager.checkAccess(job, UserGroupInformation.getCurrentUser(),
- Operation.VIEW_JOB_COUNTERS);
-
- if (!isJobInited(job)) {
- return EMPTY_COUNTERS;
- }
-
- Counters counters = job.getCounters();
- if (counters != null) {
- return new org.apache.hadoop.mapreduce.Counters(counters);
- }
- return null;
- }
+ job = jobs.get(oldJobID);
}
+ if (job != null) {
+ // check the job-access
+ aclsManager.checkAccess(job, UserGroupInformation.getCurrentUser(),
+ Operation.VIEW_JOB_COUNTERS);
+
+ if (!isJobInited(job)) {
+ return EMPTY_COUNTERS;
+ }
+
+ Counters counters = job.getCounters();
+ if (counters != null) {
+ return new org.apache.hadoop.mapreduce.Counters(counters);
+ }
+ return null;
+ }
+
Counters counters = completedJobStatusStore.readCounters(oldJobID);
if (counters != null) {
return new org.apache.hadoop.mapreduce.Counters(counters);
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskInProgress.java?rev=1066931&r1=1066930&r2=1066931&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskInProgress.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/TaskInProgress.java Thu Feb 3 20:01:09 2011
@@ -129,7 +129,7 @@ class TaskInProgress {
//task to commit, <taskattemptid>
private TaskAttemptID taskToCommit;
- private Counters counters = new Counters();
+ private volatile Counters counters = new Counters();
private HashMap<TaskAttemptID, Long> dispatchTimeMap =
new HashMap<TaskAttemptID, Long>();