You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ka...@apache.org on 2014/08/09 04:12:52 UTC
svn commit: r1616916 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-ser...
Author: kasha
Date: Sat Aug 9 02:12:52 2014
New Revision: 1616916
URL: http://svn.apache.org/r1616916
Log:
YARN-2026. Fair scheduler: Consider only active queues for computing fairshare. (Ashwin Shankar via kasha)
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
- copied unchanged from r1616915, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1616916&r1=1616915&r2=1616916&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Sat Aug 9 02:12:52 2014
@@ -82,6 +82,9 @@ Release 2.6.0 - UNRELEASED
YARN-2212. ApplicationMaster needs to find a way to update the AMRMToken
periodically. (xgong)
+ YARN-2026. Fair scheduler: Consider only active queues for computing fairshare.
+ (Ashwin Shankar via kasha)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java?rev=1616916&r1=1616915&r2=1616916&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java Sat Aug 9 02:12:52 2014
@@ -116,6 +116,18 @@ public abstract class Schedulable {
return fairShare;
}
+ /**
+ * Returns true if queue has atleast one app running. Always returns true for
+ * AppSchedulables.
+ */
+ public boolean isActive() {
+ if (this instanceof FSQueue) {
+ FSQueue queue = (FSQueue) this;
+ return queue.getNumRunnableApps() > 0;
+ }
+ return true;
+ }
+
/** Convenient toString implementation for debugging. */
@Override
public String toString() {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java?rev=1616916&r1=1616915&r2=1616916&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java Sat Aug 9 02:12:52 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
+import java.util.ArrayList;
import java.util.Collection;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -33,7 +34,31 @@ import org.apache.hadoop.yarn.server.res
public class ComputeFairShares {
private static final int COMPUTE_FAIR_SHARES_ITERATIONS = 25;
-
+
+ /**
+ * Compute fair share of the given schedulables.Fair share is an allocation of
+ * shares considering only active schedulables ie schedulables which have
+ * running apps.
+ *
+ * @param schedulables
+ * @param totalResources
+ * @param type
+ */
+ public static void computeShares(
+ Collection<? extends Schedulable> schedulables, Resource totalResources,
+ ResourceType type) {
+ Collection<Schedulable> activeSchedulables = new ArrayList<Schedulable>();
+ for (Schedulable sched : schedulables) {
+ if (sched.isActive()) {
+ activeSchedulables.add(sched);
+ } else {
+ setResourceValue(0, sched.getFairShare(), type);
+ }
+ }
+
+ computeSharesInternal(activeSchedulables, totalResources, type);
+ }
+
/**
* Given a set of Schedulables and a number of slots, compute their weighted
* fair shares. The min and max shares and of the Schedulables are assumed to
@@ -75,7 +100,7 @@ public class ComputeFairShares {
* because resourceUsedWithWeightToResourceRatio is linear-time and the number of
* iterations of binary search is a constant (dependent on desired precision).
*/
- public static void computeShares(
+ private static void computeSharesInternal(
Collection<? extends Schedulable> schedulables, Resource totalResources,
ResourceType type) {
if (schedulables.isEmpty()) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1616916&r1=1616915&r2=1616916&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Aug 9 02:12:52 2014
@@ -292,6 +292,7 @@ public class TestFairScheduler extends F
// Have two queues which want entire cluster capacity
createSchedulingRequest(10 * 1024, "queue1", "user1");
createSchedulingRequest(10 * 1024, "queue2", "user1");
+ createSchedulingRequest(10 * 1024, "root.default", "user1");
scheduler.update();
@@ -322,6 +323,7 @@ public class TestFairScheduler extends F
// Have two queues which want entire cluster capacity
createSchedulingRequest(10 * 1024, "parent.queue2", "user1");
createSchedulingRequest(10 * 1024, "parent.queue3", "user1");
+ createSchedulingRequest(10 * 1024, "root.default", "user1");
scheduler.update();
@@ -766,8 +768,10 @@ public class TestFairScheduler extends F
scheduler.handle(nodeEvent1);
// user1,user2 submit their apps to parentq and create user queues
- scheduler.assignToQueue(rmApp1, "root.parentq", "user1");
- scheduler.assignToQueue(rmApp2, "root.parentq", "user2");
+ createSchedulingRequest(10 * 1024, "root.parentq", "user1");
+ createSchedulingRequest(10 * 1024, "root.parentq", "user2");
+ // user3 submits app in default queue
+ createSchedulingRequest(10 * 1024, "root.default", "user3");
scheduler.update();
@@ -1287,7 +1291,7 @@ public class TestFairScheduler extends F
scheduler.update();
Resource toPreempt = scheduler.resToPreempt(scheduler.getQueueManager()
.getLeafQueue("queueA.queueA2", false), clock.getTime());
- assertEquals(2980, toPreempt.getMemory());
+ assertEquals(3277, toPreempt.getMemory());
// verify if the 3 containers required by queueA2 are preempted in the same
// round
@@ -2446,8 +2450,12 @@ public class TestFairScheduler extends F
scheduler.update();
FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true);
- assertEquals("Queue queue1's fair share should be 10240",
- 10240, queue1.getFairShare().getMemory());
+ assertEquals("Queue queue1's fair share should be 0", 0, queue1
+ .getFairShare().getMemory());
+
+ createSchedulingRequest(1 * 1024, "root.default", "user1");
+ scheduler.update();
+ scheduler.handle(updateEvent);
Resource amResource1 = Resource.newInstance(1024, 1);
Resource amResource2 = Resource.newInstance(2048, 2);
@@ -2635,24 +2643,32 @@ public class TestFairScheduler extends F
FSLeafQueue queue1 =
scheduler.getQueueManager().getLeafQueue("queue1", true);
- assertEquals("Queue queue1's fair share should be 1366",
- 1366, queue1.getFairShare().getMemory());
+ assertEquals("Queue queue1's fair share should be 0", 0, queue1
+ .getFairShare().getMemory());
FSLeafQueue queue2 =
scheduler.getQueueManager().getLeafQueue("queue2", true);
- assertEquals("Queue queue2's fair share should be 1366",
- 1366, queue2.getFairShare().getMemory());
+ assertEquals("Queue queue2's fair share should be 0", 0, queue2
+ .getFairShare().getMemory());
FSLeafQueue queue3 =
scheduler.getQueueManager().getLeafQueue("queue3", true);
- assertEquals("Queue queue3's fair share should be 1366",
- 1366, queue3.getFairShare().getMemory());
+ assertEquals("Queue queue3's fair share should be 0", 0, queue3
+ .getFairShare().getMemory());
FSLeafQueue queue4 =
scheduler.getQueueManager().getLeafQueue("queue4", true);
- assertEquals("Queue queue4's fair share should be 1366",
- 1366, queue4.getFairShare().getMemory());
+ assertEquals("Queue queue4's fair share should be 0", 0, queue4
+ .getFairShare().getMemory());
FSLeafQueue queue5 =
scheduler.getQueueManager().getLeafQueue("queue5", true);
- assertEquals("Queue queue5's fair share should be 1366",
- 1366, queue5.getFairShare().getMemory());
+ assertEquals("Queue queue5's fair share should be 0", 0, queue5
+ .getFairShare().getMemory());
+
+ List<String> queues = Arrays.asList("root.default", "root.queue3",
+ "root.queue4", "root.queue5");
+ for (String queue : queues) {
+ createSchedulingRequest(1 * 1024, queue, "user1");
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ }
Resource amResource1 = Resource.newInstance(2048, 1);
int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();