You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/05/03 00:49:06 UTC
svn commit: r1098825 - in /hadoop/common/branches/branch-0.20-security: ./
src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/
src/core/org/apache/hadoop/fs/ src/mapred/ src/test/org/apache/hadoop/fs/
src/tools/org/apache/hadoop/tools/
Author: omalley
Date: Mon May 2 22:49:06 2011
New Revision: 1098825
URL: http://svn.apache.org/viewvc?rev=1098825&view=rev
Log:
MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
(Todd Lipcon via cdouglas)
Modified:
hadoop/common/branches/branch-0.20-security/ (props changed)
hadoop/common/branches/branch-0.20-security/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-0.20-security/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/HarFileSystem.java (props changed)
hadoop/common/branches/branch-0.20-security/src/mapred/ (props changed)
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/fs/TestHarFileSystem.java (props changed)
hadoop/common/branches/branch-0.20-security/src/tools/org/apache/hadoop/tools/HadoopArchives.java (props changed)
Propchange: hadoop/common/branches/branch-0.20-security/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.20:826138,880632,960946
-/hadoop/common/branches/branch-0.20-security-203:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20:826138,826568,880632,960946
+/hadoop/common/branches/branch-0.20-security-203:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/core/branches/branch-0.19:713112
/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1098825&r1=1098824&r2=1098825&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Mon May 2 22:49:06 2011
@@ -1760,6 +1760,9 @@ Release 0.20.2 - Unreleased
HDFS-596. Fix memory leak in hdfsFreeFileInfo() for libhdfs.
(Zhang Bingjun via dhruba)
+ MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
+ (Todd Lipcon via cdouglas)
+
Release 0.20.1 - 2009-09-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-0.20-security/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,880632,960946
-/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20/CHANGES.txt:826138,826568,880632,960946
+/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752514,752555,752590,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
Modified: hadoop/common/branches/branch-0.20-security/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=1098825&r1=1098824&r2=1098825&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java Mon May 2 22:49:06 2011
@@ -18,7 +18,9 @@
package org.apache.hadoop.mapred;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
@@ -122,7 +124,12 @@ public class FairSchedulerServlet extend
}
// Print out the normal response
response.setContentType("text/html");
- PrintWriter out = new PrintWriter(response.getOutputStream());
+
+ // Because the client may read arbitrarily slow, and we hold locks while
+ // the servlet output, we want to write to our own buffer which we know
+ // won't block.
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter out = new PrintWriter(baos);
String hostname = StringUtils.simpleHostname(
jobTracker.getJobTrackerMachine());
out.print("<html><head>");
@@ -137,6 +144,11 @@ public class FairSchedulerServlet extend
showAdminForm(out, advancedView);
out.print("</body></html>\n");
out.close();
+
+ // Flush our buffer to the real servlet output
+ OutputStream servletOut = response.getOutputStream();
+ baos.writeTo(servletOut);
+ servletOut.close();
}
/**
@@ -206,51 +218,53 @@ public class FairSchedulerServlet extend
out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>" +
(advancedView ? "<th>Weight</th><th>Deficit</th><th>minReduces</th>" : ""));
out.print("</tr>\n");
- Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
- synchronized (scheduler) {
- for (JobInProgress job: runningJobs) {
- JobProfile profile = job.getProfile();
- JobInfo info = scheduler.infos.get(job);
- if (info == null) { // Job finished, but let's show 0's for info
- info = new JobInfo();
- }
- out.print("<tr>\n");
- out.printf("<td>%s</td>\n", DATE_FORMAT.format(
- new Date(job.getStartTime())));
- out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
- profile.getJobID(), profile.getJobID());
- out.printf("<td>%s</td>\n", profile.getUser());
- out.printf("<td>%s</td>\n", profile.getJobName());
- out.printf("<td>%s</td>\n", generateSelect(
- scheduler.getPoolManager().getPoolNames(),
- scheduler.getPoolManager().getPoolName(job),
- "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%s</td>\n", generateSelect(
- Arrays.asList(new String[]
- {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
- job.getPriority().toString(),
- "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
- (advancedView ? "&advanced" : "")));
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedMaps(), job.desiredMaps(), info.runningMaps,
- info.mapFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.mapWeight);
- out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
- (info.mapDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minMaps);
- }
- out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
- job.finishedReduces(), job.desiredReduces(), info.runningReduces,
- info.reduceFairShare);
- if (advancedView) {
- out.printf("<td>%8.1f</td>\n", info.reduceWeight);
- out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
- (info.reduceDeficit / 1000) + "s" : "--");
- out.printf("<td>%d</td>\n", info.minReduces);
+ synchronized (jobTracker) {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ synchronized (scheduler) {
+ for (JobInProgress job: runningJobs) {
+ JobProfile profile = job.getProfile();
+ JobInfo info = scheduler.infos.get(job);
+ if (info == null) { // Job finished, but let's show 0's for info
+ info = new JobInfo();
+ }
+ out.print("<tr>\n");
+ out.printf("<td>%s</td>\n", DATE_FORMAT.format(
+ new Date(job.getStartTime())));
+ out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
+ profile.getJobID(), profile.getJobID());
+ out.printf("<td>%s</td>\n", profile.getUser());
+ out.printf("<td>%s</td>\n", profile.getJobName());
+ out.printf("<td>%s</td>\n", generateSelect(
+ scheduler.getPoolManager().getPoolNames(),
+ scheduler.getPoolManager().getPoolName(job),
+ "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%s</td>\n", generateSelect(
+ Arrays.asList(new String[]
+ {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
+ job.getPriority().toString(),
+ "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
+ (advancedView ? "&advanced" : "")));
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedMaps(), job.desiredMaps(), info.runningMaps,
+ info.mapFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.mapWeight);
+ out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
+ (info.mapDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minMaps);
+ }
+ out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+ job.finishedReduces(), job.desiredReduces(), info.runningReduces,
+ info.reduceFairShare);
+ if (advancedView) {
+ out.printf("<td>%8.1f</td>\n", info.reduceWeight);
+ out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
+ (info.reduceDeficit / 1000) + "s" : "--");
+ out.printf("<td>%d</td>\n", info.minReduces);
+ }
+ out.print("</tr>\n");
}
- out.print("</tr>\n");
}
}
out.print("</table>\n");
Propchange: hadoop/common/branches/branch-0.20-security/src/core/org/apache/hadoop/fs/HarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/fs/HarFileSystem.java:1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/fs/HarFileSystem.java:1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/common/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java:910709
/hadoop/common/trunk/src/java/org/apache/hadoop/fs/HarFileSystem.java:979485
/hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/fs/HarFileSystem.java:713112
Propchange: hadoop/common/branches/branch-0.20-security/src/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.20/src/mapred:826138,880632,960946
-/hadoop/common/branches/branch-0.20-security-203/src/mapred:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20/src/mapred:826138,826568,880632,960946
+/hadoop/common/branches/branch-0.20-security-203/src/mapred:1096071,1097011,1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/core/branches/branch-0.19/src/mapred:713112
/hadoop/core/trunk/src/mapred:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
/hadoop/mapreduce/trunk/src/java:808650
Propchange: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/fs/TestHarFileSystem.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestHarFileSystem.java:979485
/hadoop/common/trunk/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:910709
/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/fs/TestHarFileSystem.java:713112
Propchange: hadoop/common/branches/branch-0.20-security/src/tools/org/apache/hadoop/tools/HadoopArchives.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 2 22:49:06 2011
@@ -1,4 +1,4 @@
-/hadoop/common/branches/branch-0.20-security-203/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1097249,1097269,1097281,1097966,1098816,1098819
+/hadoop/common/branches/branch-0.20-security-203/src/tools/org/apache/hadoop/tools/HadoopArchives.java:1097249,1097269,1097281,1097966,1098816,1098819,1098823
/hadoop/common/trunk/src/tools/org/apache/hadoop/tools/HadoopArchives.java:910709
/hadoop/core/branches/branch-0.19/src/tools/org/apache/hadoop/tools/HadoopArchives.java:713112
/hadoop/core/trunk/src/tools/org/apache/hadoop/tools/HadoopArchives.java:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569