You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2008/12/23 09:16:00 UTC
svn commit: r728903 - in /hadoop/core/branches/branch-0.20: ./ conf/
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/docs/src/documentation/content/xdocs/
Author: yhemanth
Date: Tue Dec 23 00:16:00 2008
New Revision: 728903
URL: http://svn.apache.org/viewvc?rev=728903&view=rev
Log:
Merge -r 728900:728901 from trunk to branch-0.20 to fix HADOOP-4854.
Modified:
hadoop/core/branches/branch-0.20/CHANGES.txt
hadoop/core/branches/branch-0.20/conf/capacity-scheduler.xml.template
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
hadoop/core/branches/branch-0.20/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Tue Dec 23 00:16:00 2008
@@ -488,6 +488,9 @@
HADOOP-4924. Fixes a race condition in TaskTracker re-init. (ddas)
+ HADOOP-4854. Read reclaim capacity interval from capacity scheduler
+ configuration. (Sreekanth Ramakrishnan via yhemanth)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/core/branches/branch-0.20/conf/capacity-scheduler.xml.template
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/conf/capacity-scheduler.xml.template?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/conf/capacity-scheduler.xml.template (original)
+++ hadoop/core/branches/branch-0.20/conf/capacity-scheduler.xml.template Tue Dec 23 00:16:00 2008
@@ -54,6 +54,16 @@
</description>
</property>
+
+ <property>
+ <name>mapred.capacity-scheduler.reclaimCapacity.interval</name>
+ <value>5</value>
+ <description>The time interval, in seconds, between which the scheduler
+ periodically determines whether capacity needs to be reclaimed for
+ any queue.
+ </description>
+ </property>
+
<!-- The default configuration settings for the capacity task scheduler -->
<!-- The default values would be applied to all the queues which don't have -->
<!-- the appropriate property for the particular queue -->
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Tue Dec 23 00:16:00 2008
@@ -435,4 +435,30 @@
public void setDefaultPercentOfPmemInVmem(float value) {
rmConf.setFloat(DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY, value);
}
+
+ /**
+ * Gets the reclaim capacity thread interval.
+ *
+ * @return reclaim capacity interval
+ */
+
+ public long getReclaimCapacityInterval() {
+ long reclaimCapacityInterval =
+ rmConf.getLong("mapred.capacity-scheduler.reclaimCapacity.interval", 5);
+
+ if(reclaimCapacityInterval <= 0) {
+ throw new IllegalArgumentException("Invalid reclaim capacity " +
+ "interval, should be greater than zero");
+ }
+ return reclaimCapacityInterval;
+ }
+ /**
+ * Sets the reclaim capacity thread interval.
+ *
+ * @param value
+ */
+ public void setReclaimCapacityInterval(long value) {
+ rmConf.setLong("mapred.capacity-scheduler.reclaimCapacity.interval",
+ value);
+ }
}
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Tue Dec 23 00:16:00 2008
@@ -1163,16 +1163,15 @@
public synchronized void start() throws IOException {
if (started) return;
super.start();
- RECLAIM_CAPACITY_INTERVAL =
- conf.getLong("mapred.capacity-scheduler.reclaimCapacity.interval", 5);
- RECLAIM_CAPACITY_INTERVAL *= 1000;
-
// initialize our queues from the config settings
if (null == rmConf) {
rmConf = new CapacitySchedulerConf();
}
initializeMemoryRelatedConf();
+
+ RECLAIM_CAPACITY_INTERVAL = rmConf.getReclaimCapacityInterval();
+ RECLAIM_CAPACITY_INTERVAL *= 1000;
// read queue info from config file
QueueManager queueManager = taskTrackerManager.getQueueManager();
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java Tue Dec 23 00:16:00 2008
@@ -511,6 +511,8 @@
new LinkedHashMap<String, FakeQueueInfo>();
String firstQueue;
+ private long reclaimCapacityInterval = 1000;
+
void setFakeQueues(List<FakeQueueInfo> queues) {
for (FakeQueueInfo q: queues) {
queueMap.put(q.queueName, q);
@@ -554,6 +556,16 @@
public int getMaxWorkerThreads() {
return 1;
}
+
+ @Override
+ public long getReclaimCapacityInterval() {
+ return reclaimCapacityInterval ;
+ }
+
+ @Override
+ public void setReclaimCapacityInterval(long value) {
+ this.reclaimCapacityInterval = value;
+ }
}
protected class FakeClock extends CapacityTaskScheduler.Clock {
@@ -592,8 +604,6 @@
scheduler.setTaskTrackerManager(taskTrackerManager);
conf = new JobConf();
- // set interval to a large number so thread doesn't interfere with us
- conf.setLong("mapred.capacity-scheduler.reclaimCapacity.interval", 500);
scheduler.setConf(conf);
}
@@ -1174,6 +1184,7 @@
queues.add(new FakeQueueInfo("q2", 25.0f, 1000, true, 25));
queues.add(new FakeQueueInfo("q3", 25.0f, 1000, true, 25));
resConf.setFakeQueues(queues);
+ resConf.setReclaimCapacityInterval(500);
scheduler.setResourceManagerConf(resConf);
scheduler.start();
@@ -1216,6 +1227,7 @@
queues.add(new FakeQueueInfo("q3", 20.0f, 1000, true, 25));
queues.add(new FakeQueueInfo("q4", 10.0f, 1000, true, 25));
resConf.setFakeQueues(queues);
+ resConf.setReclaimCapacityInterval(500);
scheduler.setResourceManagerConf(resConf);
scheduler.start();
@@ -1272,6 +1284,7 @@
queues.add(new FakeQueueInfo("default", 50.0f, 1000, true, 25));
queues.add(new FakeQueueInfo("queue", 50.0f, 1000, true, 25));
resConf.setFakeQueues(queues);
+ resConf.setReclaimCapacityInterval(500);
scheduler.setResourceManagerConf(resConf);
scheduler.start();
@@ -1306,6 +1319,7 @@
queues.add(new FakeQueueInfo("default", 50.0f, 1000, true, 25));
queues.add(new FakeQueueInfo("q2", 50.0f, 1000, true, 25));
resConf.setFakeQueues(queues);
+ resConf.setReclaimCapacityInterval(500);
scheduler.setResourceManagerConf(resConf);
scheduler.start();
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java Tue Dec 23 00:16:00 2008
@@ -372,6 +372,31 @@
} catch (IllegalArgumentException e) {}
}
+ public void testInvalidReclaimCapacityInterval() throws IOException {
+ openFile();
+ startConfig();
+ Map<String, String> q1Props = setupQueueProperties(
+ new String[] { "guaranteed-capacity",
+ "reclaim-time-limit",
+ "supports-priority",
+ "minimum-user-limit-percent" },
+ new String[] { "-1",
+ "-800",
+ "true",
+ "50" }
+ );
+ writeQueueDetails("default", q1Props);
+ writeProperty("mapred.capacity-scheduler.reclaimCapacity.interval", "0");
+ endConfig();
+ try {
+ testConf = new CapacitySchedulerConf(new Path(testConfFile));
+ testConf.getReclaimCapacityInterval();
+ fail("Expect Invalid reclaim capacity interval raise Exception");
+ }catch(IllegalArgumentException e) {
+ assertTrue(true);
+ }
+ }
+
private void checkQueueProperties(
CapacitySchedulerConf testConf,
Map<String, Map<String, String>> queueDetails) {
Modified: hadoop/core/branches/branch-0.20/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml?rev=728903&r1=728902&r2=728903&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml (original)
+++ hadoop/core/branches/branch-0.20/src/docs/src/documentation/content/xdocs/capacity_scheduler.xml Tue Dec 23 00:16:00 2008
@@ -234,6 +234,26 @@
</tr>
</table>
</section>
+
+ <section>
+ <title>Configuring the capacity scheduler</title>
+ <p>The capacity scheduler's behavior can be controlled through the
+ following properties.
+ </p>
+ <table>
+ <tr>
+ <th>Name</th><th>Description</th>
+ </tr>
+ <tr>
+ <td>mapred.capacity-scheduler.reclaimCapacity.interval</td>
+ <td>The time interval, in seconds, between which the scheduler
+ periodically determines whether capacity needs to be reclaimed for
+ any queue. The default value is 5 seconds.
+ </td>
+ </tr>
+ </table>
+
+ </section>
<section>
<title>Reviewing the configuration of the capacity scheduler</title>