You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2008/12/18 08:13:10 UTC
svn commit: r727663 - in /hadoop/core/branches/branch-0.20: ./
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
Author: yhemanth
Date: Wed Dec 17 23:13:09 2008
New Revision: 727663
URL: http://svn.apache.org/viewvc?rev=727663&view=rev
Log:
Merge -r 727659:727660 from trunk to branch-0.20 to fix HADOOP-4774.
Modified:
hadoop/core/branches/branch-0.20/CHANGES.txt
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=727663&r1=727662&r2=727663&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Wed Dec 17 23:13:09 2008
@@ -453,6 +453,10 @@
lib directory and added the same to test-patch.sh. (Giridharan Kesavan via
acmurthy)
+ HADOOP-4774. Fix default values of some capacity scheduler configuration
+ items which would otherwise not work on a fresh checkout.
+ (Sreekanth Ramakrishnan via yhemanth)
+
Release 0.19.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=727663&r1=727662&r2=727663&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Wed Dec 17 23:13:09 2008
@@ -71,6 +71,18 @@
static final String UPPER_LIMIT_ON_TASK_PMEM_PROPERTY =
"mapred.capacity-scheduler.task.limit.maxpmem";
+ /**
+ * The constant which defines the default initialization thread
+ * polling interval, denoted in milliseconds.
+ */
+ private static final int INITIALIZATION_THREAD_POLLING_INTERVAL = 5000;
+
+ /**
+ * The constant which defines the maximum number of worker threads to be
+ * spawned off for job initialization
+ */
+ private static final int MAX_INITIALIZATION_WORKER_THREADS = 5;
+
private Configuration rmConf;
private int defaultMaxJobsPerUsersToInitialize;
@@ -312,15 +324,19 @@
}
/**
- * Amount of time in miliseconds which poller thread and initialization
+ * Amount of time in milliseconds which poller thread and initialization
* thread would sleep before looking at the queued jobs.
+ *
+ * The default value if no corresponding configuration is present is
+ * 5000 Milliseconds.
*
- * @return time in miliseconds.
+ * @return time in milliseconds.
* @throws IllegalArgumentException if time is negative or zero.
*/
public long getSleepInterval() {
long sleepInterval = rmConf.getLong(
- "mapred.capacity-scheduler.init-poll-interval", -1);
+ "mapred.capacity-scheduler.init-poll-interval",
+ INITIALIZATION_THREAD_POLLING_INTERVAL);
if(sleepInterval <= 0) {
throw new IllegalArgumentException(
@@ -342,12 +358,15 @@
* So a given thread can have responsibility of initializing jobs from more
* than one queue.
*
+ * The default value is 5
+ *
* @return maximum number of threads spawned to initialize jobs in job queue
* in parallel.
*/
public int getMaxWorkerThreads() {
int maxWorkerThreads = rmConf.getInt(
- "mapred.capacity-scheduler.init-worker-threads", 0);
+ "mapred.capacity-scheduler.init-worker-threads",
+ MAX_INITIALIZATION_WORKER_THREADS);
if(maxWorkerThreads <= 0) {
throw new IllegalArgumentException(
"Invalid initializater worker thread number " + maxWorkerThreads);
Modified: hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java?rev=727663&r1=727662&r2=727663&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java (original)
+++ hadoop/core/branches/branch-0.20/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java Wed Dec 17 23:13:09 2008
@@ -323,6 +323,55 @@
}
}
+ public void testInitializationPollerProperties()
+ throws Exception {
+ /*
+ * Test case to check properties of poller when no configuration file
+ * is present.
+ */
+ testConf = new CapacitySchedulerConf();
+ long pollingInterval = testConf.getSleepInterval();
+ int maxWorker = testConf.getMaxWorkerThreads();
+ assertTrue("Invalid polling interval ",pollingInterval > 0);
+ assertTrue("Invalid working thread pool size" , maxWorker > 0);
+
+ //test case for custom values configured for initialization
+ //poller.
+ openFile();
+ startConfig();
+ writeProperty("mapred.capacity-scheduler.init-worker-threads", "1");
+ writeProperty("mapred.capacity-scheduler.init-poll-interval", "1");
+ endConfig();
+
+ testConf = new CapacitySchedulerConf(new Path(testConfFile));
+
+ pollingInterval = testConf.getSleepInterval();
+
+ maxWorker = testConf.getMaxWorkerThreads();
+
+ assertEquals("Invalid polling interval ",pollingInterval ,1);
+ assertEquals("Invalid working thread pool size" , maxWorker, 1);
+
+ //Test case for invalid values configured for initialization
+ //poller
+ openFile();
+ startConfig();
+ writeProperty("mapred.capacity-scheduler.init-worker-threads", "0");
+ writeProperty("mapred.capacity-scheduler.init-poll-interval", "0");
+ endConfig();
+
+ testConf = new CapacitySchedulerConf(new Path(testConfFile));
+
+ try {
+ pollingInterval = testConf.getSleepInterval();
+ fail("Polling interval configured is illegal");
+ } catch (IllegalArgumentException e) {}
+ try {
+ maxWorker = testConf.getMaxWorkerThreads();
+ fail("Max worker thread configured is illegal");
+ } catch (IllegalArgumentException e) {}
+ }
+
private void checkQueueProperties(
CapacitySchedulerConf testConf,
Map<String, Map<String, String>> queueDetails) {
@@ -396,15 +445,6 @@
}
- private void writeUserDefinedDefaultConfigurationWithoutGC() {
- writeProperty("mapred.capacity-scheduler.default-reclaim-time-limit"
- , "800");
- writeProperty("mapred.capacity-scheduler.default-supports-priority"
- , "true");
- writeProperty("mapred.capacity-scheduler.default-minimum-user-limit-percent"
- , "50");
- }
-
private void writeProperty(String name, String value) {
writer.println("<property>");
writer.println("<name> " + name + "</name>");