You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tu...@apache.org on 2013/06/03 19:33:56 UTC
svn commit: r1489070 [2/2] - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java?rev=1489070&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java Mon Jun 3 17:33:55 2013
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
+
+import static org.junit.Assert.*;
+
+import java.util.Comparator;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Test;
+
+/**
+ * comparator.compare(sched1, sched2) < 0 means that sched1 should get a
+ * container before sched2
+ */
+public class TestDominantResourceFairnessPolicy {
+
+ private Comparator<Schedulable> createComparator(int clusterMem,
+ int clusterCpu) {
+ DominantResourceFairnessPolicy policy = new DominantResourceFairnessPolicy();
+ policy.initialize(BuilderUtils.newResource(clusterMem, clusterCpu));
+ return policy.getComparator();
+ }
+
+ private Schedulable createSchedulable(int memUsage, int cpuUsage) {
+ return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL, 0, 0);
+ }
+
+ private Schedulable createSchedulable(int memUsage, int cpuUsage,
+ int minMemShare, int minCpuShare) {
+ return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL,
+ minMemShare, minCpuShare);
+ }
+
+ private Schedulable createSchedulable(int memUsage, int cpuUsage,
+ ResourceWeights weights) {
+ return createSchedulable(memUsage, cpuUsage, weights, 0, 0);
+ }
+
+
+ private Schedulable createSchedulable(int memUsage, int cpuUsage,
+ ResourceWeights weights, int minMemShare, int minCpuShare) {
+ Resource usage = BuilderUtils.newResource(memUsage, cpuUsage);
+ Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare);
+ return new FakeSchedulable(Resources.none(), minShare, weights,
+ Resources.none(), usage, 0l);
+ }
+
+ @Test
+ public void testSameDominantResource() {
+ assertTrue(createComparator(8000, 4).compare(
+ createSchedulable(1000, 1),
+ createSchedulable(2000, 1)) < 0);
+ }
+
+ @Test
+ public void testDifferentDominantResource() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(4000, 3),
+ createSchedulable(2000, 5)) < 0);
+ }
+
+ @Test
+ public void testOneIsNeedy() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(2000, 5, 0, 6),
+ createSchedulable(4000, 3, 0, 0)) < 0);
+ }
+
+ @Test
+ public void testBothAreNeedy() {
+ assertTrue(createComparator(8000, 100).compare(
+ // dominant share is 2000/8000
+ createSchedulable(2000, 5),
+ // dominant share is 4000/8000
+ createSchedulable(4000, 3)) < 0);
+ assertTrue(createComparator(8000, 100).compare(
+ // dominant min share is 2/3
+ createSchedulable(2000, 5, 3000, 6),
+ // dominant min share is 4/5
+ createSchedulable(4000, 3, 5000, 4)) < 0);
+ }
+
+ @Test
+ public void testEvenWeightsSameDominantResource() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(3000, 1, new ResourceWeights(2.0f)),
+ createSchedulable(2000, 1)) < 0);
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(1000, 3, new ResourceWeights(2.0f)),
+ createSchedulable(1000, 2)) < 0);
+ }
+
+ @Test
+ public void testEvenWeightsDifferentDominantResource() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(1000, 3, new ResourceWeights(2.0f)),
+ createSchedulable(2000, 1)) < 0);
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(3000, 1, new ResourceWeights(2.0f)),
+ createSchedulable(1000, 2)) < 0);
+ }
+
+ @Test
+ public void testUnevenWeightsSameDominantResource() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)),
+ createSchedulable(2000, 1)) < 0);
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)),
+ createSchedulable(1000, 2)) < 0);
+ }
+
+ @Test
+ public void testUnevenWeightsDifferentDominantResource() {
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)),
+ createSchedulable(2000, 1)) < 0);
+ assertTrue(createComparator(8000, 8).compare(
+ createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)),
+ createSchedulable(1000, 2)) < 0);
+ }
+
+ @Test
+ public void testCalculateShares() {
+ Resource used = Resources.createResource(10, 5);
+ Resource capacity = Resources.createResource(100, 10);
+ ResourceType[] resourceOrder = new ResourceType[2];
+ ResourceWeights shares = new ResourceWeights();
+ DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator =
+ new DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
+ comparator.calculateShares(used, capacity, shares, resourceOrder,
+ ResourceWeights.NEUTRAL);
+
+ assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
+ assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
+ assertEquals(ResourceType.CPU, resourceOrder[0]);
+ assertEquals(ResourceType.MEMORY, resourceOrder[1]);
+ }
+}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1489070&r1=1489069&r2=1489070&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon Jun 3 17:33:55 2013
@@ -31,17 +31,18 @@ Hadoop MapReduce Next Generation - Fair
* {Introduction}
Fair scheduling is a method of assigning resources to applications such that
- all apps get, on average, an equal share of resources over time.
- Hadoop NextGen is capable of scheduling multiple resource types, such as
- Memory and CPU. Currently only memory is supported, so a "cluster share" is
- a proportion of aggregate memory in the cluster. When there is a single app
- running, that app uses the entire cluster. When other apps are submitted,
- resources that free up are assigned to the new apps, so that each app gets
- roughly the same amount of resources. Unlike the default Hadoop scheduler,
- which forms a queue of apps, this lets short apps finish in reasonable time
- while not starving long-lived apps. It is also a reasonable way to share a
- cluster between a number of users. Finally, fair sharing can also work with
- app priorities - the priorities are used as weights to determine the
+ all apps get, on average, an equal share of resources over time.
+ Hadoop NextGen is capable of scheduling multiple resource types. By default,
+ the Fair Scheduler bases scheduling fairness decisions only on memory. It
+ can be configured to schedule with both memory and CPU, using the notion
+ of Dominant Resource Fairness developed by Ghodsi et al. When there is a
+ single app running, that app uses the entire cluster. When other apps are
+ submitted, resources that free up are assigned to the new apps, so that each
+ app eventually on gets roughly the same amount of resources. Unlike the default
+ Hadoop scheduler, which forms a queue of apps, this lets short apps finish in
+ reasonable time while not starving long-lived apps. It is also a reasonable way
+ to share a cluster between a number of users. Finally, fair sharing can also
+ work with app priorities - the priorities are used as weights to determine the
fraction of total resources that each app should get.
The scheduler organizes apps further into "queues", and shares resources
@@ -49,9 +50,10 @@ Hadoop MapReduce Next Generation - Fair
called âdefaultâ. If an app specifically lists a queue in a container
resource request, the request is submitted to that queue. It is also
possible to assign queues based on the user name included with the request
- through configuration. Within each queue, fair sharing is used to share
- capacity between the running apps. queues can also be given weights to share
- the cluster non-proportionally in the config file.
+ through configuration. Within each queue, a scheduling policy is used to share
+ resources between the running apps. The default is memory-based fair sharing,
+ but FIFO and multi-resource with Dominant Resource Fairness can also be
+ configured. Queues can be configured with weights to share the cluster non-evenly.
The fair scheduler supports hierarchical queues. All queues descend from a
queue named "root". Available resources are distributed among the children
@@ -120,14 +122,6 @@ Hadoop MapReduce Next Generation - Fair
queues and their properties, in addition to certain policy defaults. This file
must be in XML format as described in the next section.
- * <<<yarn.scheduler.fair.minimum-allocation-mb>>>
-
- * The smallest container size the scheduler can allocate, in MB of memory.
-
- * <<<yarn.scheduler.fair.maximum-allocation-mb>>>
-
- * The largest container the scheduler can allocate, in MB of memory.
-
* <<<yarn.scheduler.fair.user-as-default-queue>>>
* Whether to use the username associated with the allocation as the default
@@ -183,17 +177,23 @@ Allocation file format
* <<Queue elements>>, which represent queues. Each may contain the following
properties:
- * minResources: minimum MB of aggregate memory the queue expects. If a queue
- demands resources, and its current allocation is below its configured minimum,
- it will be assigned available resources before any queue that is not in this
- situation. If multiple queues are in this situation, resources go to the
- queue with the smallest ratio between allocation and minimum. Note that it is
- possible that a queue that is below its minimum may not immediately get up to
- its minimum when it submits an application, because already-running jobs may
- be using those resources.
-
- * maxResources: maximum MB of aggregate memory a queue is allowed. A queue
- will never be assigned a container that would put it over this limit.
+ * minResources: minimum resources the queue is entitled to, in the form
+ "X mb, Y vcores". If a queue's minimum share is not satisfied, it will be
+ offered available resources before any other queue under the same parent.
+ Under the single-resource fairness policy, a queue
+ is considered unsatisfied if its memory usage is below its minimum memory
+ share. Under dominant resource fairness, a queue is considered unsatisfied
+ if its usage for its dominant resource with respect to the cluster capacity
+ is below its minimum share for that resource. If multiple queues are
+ unsatisfied in this situation, resources go to the queue with the smallest
+ ratio between relevant resource usage and minimum. Note that it is
+ possible that a queue that is below its minimum may not immediately get up
+ to its minimum when it submits an application, because already-running jobs
+ may be using those resources.
+
+ * maxResources: maximum resources a queue is allowed, in the form
+ "X mb, Y vcores". A queue will never be assigned a container that would
+ put its aggregate usage over this limit.
* maxRunningApps: limit the number of apps from the queue to run at once
@@ -232,13 +232,13 @@ Allocation file format
<?xml version="1.0"?>
<allocations>
<queue name="sample_queue">
- <minResources>10000</minResources>
- <maxResources>90000</maxResources>
+ <minResources>10000 mb</minResources>
+ <maxResources>90000 mb</maxResources>
<maxRunningApps>50</maxRunningApps>
<weight>2.0</weight>
<schedulingMode>fair</schedulingMode>
<queue name="sample_sub_queue">
- <minResources>5000</minResources>
+ <minResources>5000 mb</minResources>
</queue>
</queue>
<user name="sample_user">