You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2010/04/23 11:10:42 UTC
svn commit: r937201 - in /hadoop/mapreduce/trunk: ./
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapred/ src/test/
Author: yhemanth
Date: Fri Apr 23 09:10:42 2010
New Revision: 937201
URL: http://svn.apache.org/viewvc?rev=937201&view=rev
Log:
MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev targets and also fix existing warnings. Contributed by Hong Tang.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/build.xml
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java
hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Apr 23 09:10:42 2010
@@ -553,6 +553,9 @@ Trunk (unreleased changes)
MAPREDUCE-1409. IOExceptions thrown from FIleOutputCommitter::abortTask
should cause the task to fail. (Amareshwari Sriramadasu via cdouglas)
+ MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev
+ targets and also fix existing warnings. (Hong Tang via yhemanth)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/build.xml?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/build.xml (original)
+++ hadoop/mapreduce/trunk/build.xml Fri Apr 23 09:10:42 2010
@@ -742,6 +742,7 @@
<subant target="jar">
<property name="version" value="${version}"/>
<property name="dist.dir" value="${dist.dir}"/>
+ <fileset file="${contrib.dir}/capacity-scheduler/build.xml"/>
<fileset file="${contrib.dir}/streaming/build.xml"/>
<fileset file="${contrib.dir}/gridmix/build.xml"/>
<fileset file="${contrib.dir}/mumak/build.xml"/>
@@ -773,12 +774,14 @@
<sourcePath path="${mapred.src.dir}"/>
<sourcePath path="${examples.dir}" />
<sourcePath path="${tools.src}" />
+ <sourcePath path="${basedir}/src/contrib/capacity-scheduler/src/java" />
<sourcePath path="${basedir}/src/contrib/streaming/src/java" />
<sourcePath path="${basedir}/src/contrib/gridmix/src/java" />
<sourcePath path="${basedir}/src/contrib/mumak/src/java" />
<class location="${basedir}/build/${final.name}.jar" />
<class location="${basedir}/build/${examples.final.name}.jar" />
<class location="${basedir}/build/${tools.final.name}.jar" />
+ <class location="${basedir}/build/contrib/capacity-scheduler/hadoop-${version}-capacity-scheduler.jar" />
<class location="${basedir}/build/contrib/streaming/hadoop-${version}-streaming.jar" />
<class location="${basedir}/build/contrib/gridmix/hadoop-${version}-gridmix.jar" />
<class location="${basedir}/build/contrib/mumak/hadoop-${version}-mumak.jar" />
@@ -836,6 +839,7 @@
<packageset dir="${mapred.src.dir}"/>
<packageset dir="${examples.dir}"/>
+ <packageset dir="src/contrib/capacity-scheduler/src/java"/>
<packageset dir="src/contrib/streaming/src/java"/>
<packageset dir="src/contrib/data_join/src/java"/>
<packageset dir="src/contrib/index/src/java"/>
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Fri Apr 23 09:10:42 2010
@@ -578,16 +578,20 @@ class CapacityTaskScheduler extends Task
@Override
Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
throws IOException {
- ClusterStatus clusterStatus =
- scheduler.taskTrackerManager.getClusterStatus();
- int numTaskTrackers = clusterStatus.getTaskTrackers();
- return job.obtainNewMapTask(taskTracker, numTaskTrackers,
- scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+ synchronized (scheduler) {
+ ClusterStatus clusterStatus = scheduler.taskTrackerManager
+ .getClusterStatus();
+ int numTaskTrackers = clusterStatus.getTaskTrackers();
+ return job.obtainNewMapTask(taskTracker, numTaskTrackers,
+ scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+ }
}
@Override
int getClusterCapacity() {
- return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
+ synchronized (scheduler) {
+ return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
+ }
}
@Override
@@ -621,17 +625,21 @@ class CapacityTaskScheduler extends Task
@Override
Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
throws IOException {
- ClusterStatus clusterStatus =
- scheduler.taskTrackerManager.getClusterStatus();
- int numTaskTrackers = clusterStatus.getTaskTrackers();
- return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
- scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+ synchronized (scheduler) {
+ ClusterStatus clusterStatus = scheduler.taskTrackerManager
+ .getClusterStatus();
+ int numTaskTrackers = clusterStatus.getTaskTrackers();
+ return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
+ scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+ }
}
@Override
int getClusterCapacity() {
- return scheduler.taskTrackerManager.getClusterStatus()
- .getMaxReduceTasks();
+ synchronized (scheduler) {
+ return scheduler.taskTrackerManager.getClusterStatus()
+ .getMaxReduceTasks();
+ }
}
@Override
@@ -863,7 +871,7 @@ class CapacityTaskScheduler extends Task
}
/** mostly for testing purposes */
- void setInitializationPoller(JobInitializationPoller p) {
+ synchronized void setInitializationPoller(JobInitializationPoller p) {
this.initializationPoller = p;
}
@@ -888,7 +896,7 @@ class CapacityTaskScheduler extends Task
* provided for the test classes
* lets you update the QSI objects and sorted collections
*/
- void updateContextInfoForTests() {
+ synchronized void updateContextInfoForTests() {
ClusterStatus c = taskTrackerManager.getClusterStatus();
int mapClusterCapacity = c.getMaxMapTasks();
int reduceClusterCapacity = c.getMaxReduceTasks();
@@ -1011,7 +1019,7 @@ class CapacityTaskScheduler extends Task
return jobCollection;
}
- JobInitializationPoller getInitializationPoller() {
+ synchronized JobInitializationPoller getInitializationPoller() {
return initializationPoller;
}
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java Fri Apr 23 09:10:42 2010
@@ -191,6 +191,7 @@ public class JobInitializationPoller ext
if (jobs == null) {
LOG.error("Invalid queue passed to the thread : " + queue
+ " For job :: " + job.getJobID());
+ return;
}
synchronized (jobs) {
JobSchedulingInfo schedInfo = new JobSchedulingInfo(job);
@@ -212,14 +213,12 @@ public class JobInitializationPoller ext
* jobs allowed to be initialize per user in the queue.
*
*/
- private class QueueInfo {
- String queue;
+ private static class QueueInfo {
int maxUsersAllowedToInitialize;
int maxJobsPerUserToInitialize;
- public QueueInfo(String queue, int maxUsersAllowedToInitialize,
+ public QueueInfo(int maxUsersAllowedToInitialize,
int maxJobsPerUserToInitialize) {
- this.queue = queue;
this.maxJobsPerUserToInitialize = maxJobsPerUserToInitialize;
this.maxUsersAllowedToInitialize = maxUsersAllowedToInitialize;
}
@@ -290,8 +289,7 @@ public class JobInitializationPoller ext
int maxJobsPerUserToInitialize =
schedulerConf.getMaxJobsPerUserToInitialize(queue);
QueueInfo qi =
- new QueueInfo(queue, maxUsersToInitialize,
- maxJobsPerUserToInitialize);
+ new QueueInfo(maxUsersToInitialize, maxJobsPerUserToInitialize);
jobQueues.put(queue, qi);
}
sleepInterval = schedulerConf.getSleepInterval();
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java Fri Apr 23 09:10:42 2010
@@ -161,18 +161,17 @@ class JobQueue extends AbstractQueue {
j.getProfile().getUser(),
i.intValue() + numReduceSlotsForThisJob);
if (LOG.isDebugEnabled()) {
- LOG.debug(
- String.format(
- "updateQSI: job %s: run(m)=%d, "
+ synchronized (j) {
+ LOG.debug(String.format("updateQSI: job %s: run(m)=%d, "
+ "occupied(m)=%d, run(r)=%d, occupied(r)=%d, finished(m)=%d,"
+ " finished(r)=%d, failed(m)=%d, failed(r)=%d, "
- + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j
- .getJobID().toString(), numMapsRunningForThisJob,
- numMapSlotsForThisJob, numReducesRunningForThisJob,
- numReduceSlotsForThisJob, j
- .finishedMaps(), j.finishedReduces(), j.failedMapTasks,
- j.failedReduceTasks, j.speculativeMapTasks, j.speculativeReduceTasks,
- j.numMapTasks, j.numReduceTasks));
+ + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j.getJobID()
+ .toString(), numMapsRunningForThisJob, numMapSlotsForThisJob,
+ numReducesRunningForThisJob, numReduceSlotsForThisJob, j
+ .finishedMaps(), j.finishedReduces(), j.failedMapTasks,
+ j.failedReduceTasks, j.speculativeMapTasks,
+ j.speculativeReduceTasks, j.numMapTasks, j.numReduceTasks));
+ }
}
/*
@@ -355,8 +354,6 @@ class JobQueue extends AbstractQueue {
}
public void jobUpdated(JobChangeEvent event) {
- JobInProgress job = event.getJobInProgress();
-
// Check if this is the status change
if (event instanceof JobStatusChangeEvent) {
jobStateChanged((JobStatusChangeEvent) event);
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java Fri Apr 23 09:10:42 2010
@@ -101,12 +101,12 @@ public class QueueSchedulingContext {
QueueSchedulingContext(
String queueName, float capacityPercent, float maxCapacityPercent,
int ulMin) {
- this.setQueueName(new String(queueName));
+ this.setQueueName(queueName);
this.setCapacityPercent(capacityPercent);
this.setMaxCapacityPercent(maxCapacityPercent);
this.setUlMin(ulMin);
- this.setMapTSC(new TaskSchedulingContext(TaskType.MAP));
- this.setReduceTSC(new TaskSchedulingContext(TaskType.REDUCE));
+ this.setMapTSC(new TaskSchedulingContext());
+ this.setReduceTSC(new TaskSchedulingContext());
}
/**
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java Fri Apr 23 09:10:42 2010
@@ -29,8 +29,6 @@ import org.apache.hadoop.mapreduce.TaskT
*
*/
abstract class TaskDataView {
- TaskType type;
-
abstract int getRunningTasks(JobInProgress job);
abstract int getPendingTasks(JobInProgress job);
@@ -80,10 +78,7 @@ abstract class TaskDataView {
* The data view for map tasks
*/
static class MapTaskDataView extends TaskDataView {
- TaskType type;
-
MapTaskDataView() {
- type = TaskType.MAP;
}
@Override
@@ -118,10 +113,7 @@ abstract class TaskDataView {
* The data view for reduce tasks
*/
static class ReduceTaskDataView extends TaskDataView {
- TaskType type;
-
ReduceTaskDataView() {
- type = TaskType.REDUCE;
}
@Override
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java Fri Apr 23 09:10:42 2010
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred;
import org.apache.hadoop.mapreduce.TaskType;
+import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
@@ -40,8 +41,6 @@ import java.util.Set;
* ********************************************************************
*/
public class TaskSchedulingContext {
-
- private TaskType type;
/**
* the actual capacity, which depends on how many slots are available
* in the cluster at any given time.
@@ -67,10 +66,6 @@ public class TaskSchedulingContext {
" %s running reduce tasks using %d reduce slots." +
" %d additional slots reserved.";
- public TaskSchedulingContext(TaskType type) {
- this.type = type;
- }
-
/**
* reset the variables associated with tasks
*/
@@ -188,11 +183,12 @@ public class TaskSchedulingContext {
}
private void updateNoOfSlotsOccupiedByUser(Map<String, Integer> nou) {
- Set<String> keys = nou.keySet();
- for (String key : keys) {
- if (this.numSlotsOccupiedByUser.containsKey(key)) {
- int currentVal = this.numSlotsOccupiedByUser.get(key);
- this.numSlotsOccupiedByUser.put(key, currentVal + nou.get(key));
+ for (Iterator<Map.Entry<String, Integer>> it = nou.entrySet().iterator(); it.hasNext(); ) {
+ Map.Entry<String, Integer> entry = it.next();
+ String key = entry.getKey();
+ Integer currentVal = numSlotsOccupiedByUser.get(key);
+ if (currentVal != null) {
+ this.numSlotsOccupiedByUser.put(key, currentVal + entry.getValue());
}
}
}
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java Fri Apr 23 09:10:42 2010
@@ -339,7 +339,7 @@ public class CapacityTestUtils {
}
@Override
- public Task obtainNewMapTask(
+ public synchronized Task obtainNewMapTask(
final TaskTrackerStatus tts, int clusterSize,
int ignored) throws IOException {
boolean areAllMapsRunning = (mapTaskCtr == numMapTasks);
@@ -385,7 +385,7 @@ public class CapacityTestUtils {
}
@Override
- public Task obtainNewReduceTask(
+ public synchronized Task obtainNewReduceTask(
final TaskTrackerStatus tts,
int clusterSize, int ignored) throws IOException {
boolean areAllReducesRunning = (redTaskCtr == numReduceTasks);
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java Fri Apr 23 09:10:42 2010
@@ -91,6 +91,18 @@ public class TestRefreshOfQueues {
}
/**
+ * Helper method that ensures TaskScheduler is locked before calling
+ * {@link QueueManager#refreshQueues(Configuration,
+ * org.apache.hadoop.mapred.TaskScheduler.QueueRefresher)}.
+ */
+ private static void refreshQueues(QueueManager qm, Configuration conf,
+ TaskScheduler ts) throws IOException {
+ synchronized (ts) {
+ qm.refreshQueues(conf, ts.getQueueRefresher());
+ }
+ }
+
+ /**
* @throws Throwable
*/
@Test
@@ -160,8 +172,7 @@ public class TestRefreshOfQueues {
queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
// Now do scheduler refresh.
- taskTrackerManager.getQueueManager().refreshQueues(null,
- scheduler.getQueueRefresher());
+ refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
allQueues = getAllQueues(scheduler);
@@ -251,9 +262,7 @@ public class TestRefreshOfQueues {
QueueManagerTestUtils.writeQueueConfigurationFile(
queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
- taskTrackerManager.getQueueManager().refreshQueues(
- null,
- scheduler.getQueueRefresher());
+ refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
job1 =
taskTrackerManager.submitJobAndInit(
@@ -350,8 +359,7 @@ public class TestRefreshOfQueues {
queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
try {
- taskTrackerManager.getQueueManager().refreshQueues(null,
- scheduler.getQueueRefresher());
+ refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
} catch (IOException ioe) {
assertTrue(ioe.getMessage().contains(
String.format(QueueHierarchyBuilder.TOTAL_CAPACITY_OVERFLOWN_MSG,
@@ -422,9 +430,7 @@ public class TestRefreshOfQueues {
QueueManagerTestUtils.writeQueueConfigurationFile(
queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
- taskTrackerManager.getQueueManager().refreshQueues(
- null,
- scheduler.getQueueRefresher());
+ refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
job1 =
taskTrackerManager.submitJobAndInit(
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Apr 23 09:10:42 2010
@@ -4298,10 +4298,12 @@ public class JobTracker implements MRCon
public void refreshQueues() throws IOException{
LOG.info("Refreshing queue information. requested by : " +
UserGroupInformation.getCurrentUser().getShortUserName());
- this.queueManager.refreshQueues(new Configuration(this.conf),
- taskScheduler.getQueueRefresher());
+ synchronized (taskScheduler) {
+ queueManager.refreshQueues(new Configuration(this.conf), taskScheduler
+ .getQueueRefresher());
+ }
}
-
+
private void initializeTaskMemoryRelatedConfig() {
memSizeForMapSlotOnJT =
JobConf.normalizeMemoryConfigValue(conf.getLong(
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java Fri Apr 23 09:10:42 2010
@@ -99,6 +99,9 @@ abstract class TaskScheduler implements
* {@link QueueManager} whenever it gets a request from an administrator to
* refresh its own queue-configuration. This method has a documented contract
* between the {@link QueueManager} and the {@link TaskScheduler}.
+ *
+ * Before calling QueueRefresher, the caller must hold the lock to the
+ * corresponding {@link TaskScheduler} (generally in the {@link JobTracker}).
*/
abstract class QueueRefresher {
Modified: hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml Fri Apr 23 09:10:42 2010
@@ -360,4 +360,24 @@
<Method name="getHexDigit"/>
<Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
</Match>
+
+ <!--
+ False positive of FindBugs complaining that initializationPoller and
+ started were accessed through
+ CapacitySchedulerQueueRefresher.refreshQueues without explicitly holding
+ the lock of CapacityTaskScheduler. The lock is in fact acquired by
+ JobTracker prior to calling QueueManager.refereshQueues - this is
+ necessary to ensure the order of locking (TaskScheduler ->
+ QueueManager).
+ -->
+ <Match>
+ <Class name="org.apache.hadoop.mapred.CapacityTaskScheduler" />
+ <Field name="initializationPoller" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.CapacityTaskScheduler" />
+ <Field name="started" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
</FindBugsFilter>