You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:34:54 UTC
svn commit: r1077603 - in
/hadoop/common/branches/branch-0.20-security-patches:
conf/capacity-scheduler.xml.template
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
Author: omalley
Date: Fri Mar 4 04:34:53 2011
New Revision: 1077603
URL: http://svn.apache.org/viewvc?rev=1077603&view=rev
Log:
commit 9896008ed7bd91164e6916d6ff6417247b9e2b22
Author: Arun C Murthy <ac...@apache.org>
Date: Tue Jul 27 14:33:44 2010 -0700
MAPREDUCE-1872. Fixed javadoc and added sample configs in conf/capacity-scheduler.xml.template
Modified:
hadoop/common/branches/branch-0.20-security-patches/conf/capacity-scheduler.xml.template
hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
Modified: hadoop/common/branches/branch-0.20-security-patches/conf/capacity-scheduler.xml.template
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/conf/capacity-scheduler.xml.template?rev=1077603&r1=1077602&r2=1077603&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/conf/capacity-scheduler.xml.template (original)
+++ hadoop/common/branches/branch-0.20-security-patches/conf/capacity-scheduler.xml.template Fri Mar 4 04:34:53 2011
@@ -8,6 +8,14 @@
<configuration>
<property>
+ <name>mapred.capacity-scheduler.maximum-system-jobs</name>
+ <value>3000</value>
+ <description>Maximum number of jobs in the system which can be initialized,
+ concurrently, by the CapacityScheduler.
+ </description>
+ </property>
+
+ <property>
<name>mapred.capacity-scheduler.queue.default.capacity</name>
<value>100</value>
<description>Percentage of the number of slots in the cluster that are
@@ -57,11 +65,38 @@
value of 100 implies no user limits are imposed.
</description>
</property>
+
<property>
- <name>mapred.capacity-scheduler.queue.default.maximum-initialized-jobs-per-user</name>
- <value>2</value>
- <description>The maximum number of jobs to be pre-initialized for a user
- of the job queue.
+ <name>mapred.capacity-scheduler.queue.default.user-limit-factor</name>
+ <value>1</value>
+ <description>The multiple of the queue capacity which can be configured to
+ allow a single user to acquire more slots.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks</name>
+ <value>200000</value>
+ <description>The maximum number of tasks, across all jobs in the queue,
+ which can be initialized concurrently. Once the queue's jobs exceed this
+ limit they will be queued on disk.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user</name>
+ <value>100000</value>
+ <description>The maximum number of tasks per-user, across all the of the
+ user's jobs in the queue, which can be initialized concurrently. Once the
+ user's jobs exceed this limit they will be queued on disk.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.capacity-scheduler.queue.default.init-accept-jobs-factor</name>
+ <value>10</value>
+ <description>The multipe of (maximum-system-jobs * queue-capacity) used to
+ determine the number of jobs which are accepted by the scheduler.
</description>
</property>
@@ -84,14 +119,40 @@
</description>
</property>
+
+ <property>
+ <name>mapred.capacity-scheduler.default-user-limit-factor</name>
+ <value>1</value>
+ <description>The default multiple of queue-capacity which is used to
+ determine the amount of slots a single user can consume concurrently.
+ </description>
+ </property>
+
<property>
- <name>mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user</name>
- <value>2</value>
- <description>The maximum number of jobs to be pre-initialized for a user
- of the job queue.
+ <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-queue</name>
+ <value>200000</value>
+ <description>The default maximum number of tasks, across all jobs in the
+ queue, which can be initialized concurrently. Once the queue's jobs exceed
+ this limit they will be queued on disk.
</description>
</property>
+ <property>
+ <name>mapred.capacity-scheduler.default-maximum-active-tasks-per-user</name>
+ <value>100000</value>
+ <description>The default maximum number of tasks per-user, across all the of
+ the user's jobs in the queue, which can be initialized concurrently. Once
+ the user's jobs exceed this limit they will be queued on disk.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.capacity-scheduler.default-init-accept-jobs-factor</name>
+ <value>10</value>
+ <description>The default multipe of (maximum-system-jobs * queue-capacity)
+ used to determine the number of jobs which are accepted by the scheduler.
+ </description>
+ </property>
<!-- Capacity scheduler Job Initialization configuration parameters -->
<property>
Modified: hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=1077603&r1=1077602&r2=1077603&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Fri Mar 4 04:34:53 2011
@@ -90,7 +90,7 @@ class CapacitySchedulerConf {
* The constant which defines the default initialization thread
* polling interval, denoted in milliseconds.
*/
- private static final int INITIALIZATION_THREAD_POLLING_INTERVAL = 5000;
+ private static final int INITIALIZATION_THREAD_POLLING_INTERVAL = 3000;
/**
* The constant which defines the maximum number of worker threads to be
@@ -386,7 +386,7 @@ class CapacitySchedulerConf {
}
/**
- * Get the maximum active tasks per user to be initialized.
+ * Get the maximum active tasks per queue to be initialized.
*
* @param queue queue name
*/
@@ -397,7 +397,7 @@ class CapacitySchedulerConf {
}
/**
- * Set the maximum active tasks per user to be initialized.
+ * Set the maximum active tasks per queue to be initialized.
*
* @param queue queue name
* @param value maximum active tasks
@@ -408,7 +408,7 @@ class CapacitySchedulerConf {
}
/**
- * Get the maximum active tasks per user to be initialized.
+ * Get the maximum active tasks per-user, per-queue to be initialized.
*
* @param queue queue name
*/
@@ -419,7 +419,7 @@ class CapacitySchedulerConf {
}
/**
- * Set the maximum active tasks per user to be initialized.
+ * Set the maximum active tasks per-user, per-queue to be initialized.
*
* @param queue queue name
* @param value maximum active tasks