You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2009/05/05 07:59:00 UTC
svn commit: r771570 - in /hadoop/core/trunk: CHANGES.txt
src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Author: matei
Date: Tue May 5 05:59:00 2009
New Revision: 771570
URL: http://svn.apache.org/viewvc?rev=771570&view=rev
Log:
HADOOP-5745. Allow setting the default value of maxRunningJobs for all
pools. Contributed by Dhruba Borthakur.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May 5 05:59:00 2009
@@ -110,6 +110,9 @@
HADOOP-5266. Adds the capability to do mark/reset of the reduce values
iterator in the Context object API. (Jothi Padmanabhan via ddas)
+ HADOOP-5745. Allow setting the default value of maxRunningJobs for all
+ pools. (dhruba via matei)
+
IMPROVEMENTS
HADOOP-4565. Added CombineFileInputFormat to use data locality information
Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java (original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java Tue May 5 05:59:00 2009
@@ -70,6 +70,7 @@
private Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
private Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
private int userMaxJobsDefault = Integer.MAX_VALUE;
+ private int poolMaxJobsDefault = Integer.MAX_VALUE;
private String allocFile; // Path to XML file containing allocations
private String poolNameProperty; // Jobconf property to use for determining a
@@ -168,6 +169,7 @@
Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
Map<String, Double> poolWeights = new HashMap<String, Double>();
int userMaxJobsDefault = Integer.MAX_VALUE;
+ int poolMaxJobsDefault = Integer.MAX_VALUE;
// Remember all pool names so we can display them on web UI, etc.
List<String> poolNamesInAllocFile = new ArrayList<String>();
@@ -233,6 +235,10 @@
String text = ((Text)element.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
userMaxJobsDefault = val;
+ } else if ("poolMaxJobsDefault".equals(element.getTagName())) {
+ String text = ((Text)element.getFirstChild()).getData().trim();
+ int val = Integer.parseInt(text);
+ poolMaxJobsDefault = val;
} else {
LOG.warn("Bad element in allocations file: " + element.getTagName());
}
@@ -246,6 +252,7 @@
this.poolMaxJobs = poolMaxJobs;
this.userMaxJobs = userMaxJobs;
this.userMaxJobsDefault = userMaxJobsDefault;
+ this.poolMaxJobsDefault = poolMaxJobsDefault;
this.poolWeights = poolWeights;
for (String name: poolNamesInAllocFile) {
getPool(name);
@@ -328,7 +335,7 @@
if (poolMaxJobs.containsKey(pool)) {
return poolMaxJobs.get(pool);
} else {
- return Integer.MAX_VALUE;
+ return poolMaxJobsDefault;
}
}
Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java (original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java Tue May 5 05:59:00 2009
@@ -316,6 +316,8 @@
out.println("<pool name=\"poolD\">");
out.println("<maxRunningJobs>3</maxRunningJobs>");
out.println("</pool>");
+ // Set default limit of jobs per pool to 15
+ out.println("<poolMaxJobsDefault>15</poolMaxJobsDefault>");
// Set default limit of jobs per user to 5
out.println("<userMaxJobsDefault>5</userMaxJobsDefault>");
// Give user1 a limit of 10 jobs
@@ -341,7 +343,9 @@
assertEquals(0, poolManager.getAllocation("poolC", TaskType.REDUCE));
assertEquals(0, poolManager.getAllocation("poolD", TaskType.MAP));
assertEquals(0, poolManager.getAllocation("poolD", TaskType.REDUCE));
- assertEquals(Integer.MAX_VALUE, poolManager.getPoolMaxJobs("poolA"));
+ assertEquals(15, poolManager.getPoolMaxJobs("poolA"));
+ assertEquals(15, poolManager.getPoolMaxJobs("poolB"));
+ assertEquals(15, poolManager.getPoolMaxJobs("poolC"));
assertEquals(3, poolManager.getPoolMaxJobs("poolD"));
assertEquals(10, poolManager.getUserMaxJobs("user1"));
assertEquals(5, poolManager.getUserMaxJobs("user2"));