You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/02/05 01:44:40 UTC
svn commit: r1067363 - in /hadoop/mapreduce/branches/branch-0.22: ./
src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapred/
Author: shv
Date: Sat Feb 5 00:44:40 2011
New Revision: 1067363
URL: http://svn.apache.org/viewvc?rev=1067363&view=rev
Log:
MAPREDUCE-1825. Merge r1067361 from trunk to branch 0.22.
Modified:
hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java
Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Sat Feb 5 00:44:40 2011
@@ -475,6 +475,10 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2304. Fix TestMRCLI to allow hostname with a hyphen (-).
(Priyo Mustafi via shv)
+ MAPREDUCE-1825. jobqueue_details.jsp and FairSchedulerServelet should not
+ call finishedMaps and finishedReduces when job is not initialized.
+ (Scott Chen via shv)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java Sat Feb 5 00:44:40 2011
@@ -676,6 +676,9 @@ public class FairScheduler extends TaskS
Map<String, Integer> userJobs = new HashMap<String, Integer>();
Map<String, Integer> poolJobs = new HashMap<String, Integer>();
for (JobInProgress job: jobs) {
+ if (!job.inited()) {
+ continue;
+ }
String user = job.getJobConf().getUser();
String pool = poolMgr.getPoolName(job);
int userCount = userJobs.containsKey(user) ? userJobs.get(user) : 0;
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java Sat Feb 5 00:44:40 2011
@@ -30,6 +30,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
import javax.servlet.ServletContext;
@@ -83,7 +84,7 @@ public class FairSchedulerServlet extend
boolean advancedView = request.getParameter("advanced") != null;
if (JSPUtil.privateActionsAllowed(jobTracker.conf)
&& request.getParameter("setPool") != null) {
- Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ Collection<JobInProgress> runningJobs = getInitedJobs();
PoolManager poolMgr = null;
synchronized (scheduler) {
poolMgr = scheduler.getPoolManager();
@@ -104,7 +105,7 @@ public class FairSchedulerServlet extend
}
if (JSPUtil.privateActionsAllowed(jobTracker.conf)
&& request.getParameter("setPriority") != null) {
- Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ Collection<JobInProgress> runningJobs = getInitedJobs();
JobPriority priority = JobPriority.valueOf(request.getParameter(
"setPriority"));
String jobId = request.getParameter("jobid");
@@ -246,7 +247,7 @@ public class FairSchedulerServlet extend
(advancedView ? "<th>Weight</th>" : ""));
out.print("</tr>\n");
synchronized (jobTracker) {
- Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ Collection<JobInProgress> runningJobs = getInitedJobs();
synchronized (scheduler) {
for (JobInProgress job: runningJobs) {
JobProfile profile = job.getProfile();
@@ -323,4 +324,19 @@ public class FairSchedulerServlet extend
html.append("</select>\n");
return html.toString();
}
+
+ /**
+ * Obtained all initialized jobs
+ */
+ private Collection<JobInProgress> getInitedJobs() {
+ Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+ for (Iterator<JobInProgress> it = runningJobs.iterator(); it.hasNext();) {
+ JobInProgress job = it.next();
+ if (!job.inited()) {
+ it.remove();
+ }
+ }
+ return runningJobs;
+ }
+
}
Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java Sat Feb 5 00:44:40 2011
@@ -99,6 +99,9 @@ public class JobSchedulable extends Sche
@Override
public int getRunningTasks() {
+ if (!job.inited()) {
+ return 0;
+ }
return taskType == TaskType.MAP ? job.runningMaps() : job.runningReduces();
}
Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java Sat Feb 5 00:44:40 2011
@@ -264,6 +264,15 @@ class JSPUtil {
public static String generateJobTable(String label, Collection<JobInProgress> jobs
, int refresh, int rowId, JobConf conf) throws IOException {
+ // Remove uninitialized jobs because calling JobInProgress synchronized
+ // methods while job initialization takes long time
+ for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext();) {
+ JobInProgress job = it.next();
+ if (!job.inited()) {
+ it.remove();
+ }
+ }
+
boolean isModifiable = label.equals("Running") &&
privateActionsAllowed(conf);
StringBuilder sb = new StringBuilder();