You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2009/02/25 15:56:48 UTC
svn commit: r747802 - in /hadoop/core/trunk: CHANGES.txt
src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Author: yhemanth
Date: Wed Feb 25 14:56:48 2009
New Revision: 747802
URL: http://svn.apache.org/viewvc?rev=747802&view=rev
Log:
HADOOP-5154. Fixes a deadlock in the fairshare scheduler. Contributed by Matei Zaharia.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=747802&r1=747801&r2=747802&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Feb 25 14:56:48 2009
@@ -887,6 +887,13 @@
HADOOP-5272. Fixes a problem to do with detecting whether an attempt is the first
attempt of a Task. This affects JobTracker restart. (Amar Kamat via ddas)
+Release 0.19.2 - Unreleased
+
+ BUG FIXES
+
+ HADOOP-5154. Fixes a deadlock in the fairshare scheduler.
+ (Matei Zaharia via yhemanth)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=747802&r1=747801&r2=747802&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java (original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java Wed Feb 25 14:56:48 2009
@@ -708,11 +708,11 @@
clusterStatus.getMaxMapTasks() : clusterStatus.getMaxReduceTasks());
}
- public boolean getUseFifo() {
+ public synchronized boolean getUseFifo() {
return useFifo;
}
- public void setUseFifo(boolean useFifo) {
+ public synchronized void setUseFifo(boolean useFifo) {
this.useFifo = useFifo;
}
Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=747802&r1=747801&r2=747802&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java (original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java Wed Feb 25 14:56:48 2009
@@ -86,11 +86,11 @@
return;
}
if (request.getParameter("setPool") != null) {
- PoolManager poolMgr = scheduler.getPoolManager();
- synchronized (poolMgr) {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
+ PoolManager poolMgr = scheduler.getPoolManager();
String pool = request.getParameter("setPool");
String jobId = request.getParameter("jobid");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
for (JobInProgress job: runningJobs) {
if (job.getProfile().getJobID().toString().equals(jobId)) {
poolMgr.setPool(job, pool);
@@ -103,12 +103,11 @@
return;
}
if (request.getParameter("setPriority") != null) {
- PoolManager poolMgr = scheduler.getPoolManager();
- synchronized (poolMgr) {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
JobPriority priority = JobPriority.valueOf(request.getParameter(
"setPriority"));
String jobId = request.getParameter("jobid");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
for (JobInProgress job: runningJobs) {
if (job.getProfile().getJobID().toString().equals(jobId)) {
job.setPriority(priority);
@@ -127,8 +126,6 @@
jobTracker.getJobTrackerMachine());
out.print("<html><head>");
out.printf("<title>%s Job Scheduler Admininstration</title>\n", hostname);
- out.printf("<META http-equiv=\"refresh\" content=\"15;URL=/scheduler%s\">",
- advancedView ? "?advanced" : "");
out.print("<link rel=\"stylesheet\" type=\"text/css\" " +
"href=\"/static/hadoop.css\">\n");
out.print("</head><body>\n");
@@ -145,8 +142,8 @@
* Print a view of pools to the given output writer.
*/
private void showPools(PrintWriter out, boolean advancedView) {
- PoolManager poolManager = scheduler.getPoolManager();
- synchronized(poolManager) {
+ synchronized(scheduler) {
+ PoolManager poolManager = scheduler.getPoolManager();
out.print("<h2>Pools</h2>\n");
out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
out.print("<tr><th>Pool</th><th>Running Jobs</th>" +
@@ -208,50 +205,52 @@
out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>" +
(advancedView ? "<th>Weight</th><th>Deficit</th><th>minReduces</th>" : ""));
out.print("</tr>\n");
- Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
- for (JobInProgress job: runningJobs) {
- JobProfile profile = job.getProfile();
- JobInfo info = scheduler.infos.get(job);
- if (info == null) { // Job finished, but let's show 0's for info
- info = new JobInfo();
- }
- out.print("<tr>\n");
- out.printf("<td>%s</td>\n", DATE_FORMAT.format(
- new Date(job.getStartTime())));
- out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
- profile.getJobID(), profile.getJobID());
- out.printf("<td>%s</td>\n", profile.getUser());
- out.printf("<td>%s</td>\n", profile.getJobName());
- out.printf("<td>%s</td>\n", generateSelect(
- scheduler.getPoolManager().getPoolNames(),
- scheduler.getPoolManager().getPoolName(job),
- "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%s</td>\n", generateSelect(
- Arrays.asList(new String[]
- {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
- job.getPriority().toString(),
- "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedMaps(), job.desiredMaps(), info.runningMaps,
- info.mapFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.mapWeight);
- out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
- (info.mapDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minMaps);
- }
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedReduces(), job.desiredReduces(), info.runningReduces,
- info.reduceFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.reduceWeight);
- out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
- (info.reduceDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minReduces);
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
+ for (JobInProgress job: runningJobs) {
+ JobProfile profile = job.getProfile();
+ JobInfo info = scheduler.infos.get(job);
+ if (info == null) { // Job finished, but let's show 0's for info
+ info = new JobInfo();
+ }
+ out.print("<tr>\n");
+ out.printf("<td>%s</td>\n", DATE_FORMAT.format(
+ new Date(job.getStartTime())));
+ out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
+ profile.getJobID(), profile.getJobID());
+ out.printf("<td>%s</td>\n", profile.getUser());
+ out.printf("<td>%s</td>\n", profile.getJobName());
+ out.printf("<td>%s</td>\n", generateSelect(
+ scheduler.getPoolManager().getPoolNames(),
+ scheduler.getPoolManager().getPoolName(job),
+ "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%s</td>\n", generateSelect(
+ Arrays.asList(new String[]
+ {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
+ job.getPriority().toString(),
+ "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedMaps(), job.desiredMaps(), info.runningMaps,
+ info.mapFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.mapWeight);
+ out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
+ (info.mapDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minMaps);
+ }
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedReduces(), job.desiredReduces(), info.runningReduces,
+ info.reduceFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.reduceWeight);
+ out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
+ (info.reduceDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minReduces);
+ }
+ out.print("</tr>\n");
}
- out.print("</tr>\n");
}
out.print("</table>\n");
}