You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2013/01/15 17:12:34 UTC
svn commit: r1433484 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: tomwhite
Date: Tue Jan 15 16:12:33 2013
New Revision: 1433484
URL: http://svn.apache.org/viewvc?rev=1433484&view=rev
Log:
YARN-335. Fair scheduler doesn't check whether rack needs containers before assigning to node. Contributed by Sandy Ryza.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1433484&r1=1433483&r2=1433484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Tue Jan 15 16:12:33 2013
@@ -189,6 +189,9 @@ Release 2.0.3-alpha - Unreleased
YARN-330. Fix flakey test: TestNodeManagerShutdown#testKillContainersOnShutdown.
(Sandy Ryza via hitesh)
+
+ YARN-335. Fair scheduler doesn't check whether rack needs containers
+ before assigning to node. (Sandy Ryza via tomwhite)
Release 2.0.2-alpha - 2012-09-07
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1433484&r1=1433483&r2=1433484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Tue Jan 15 16:12:33 2013
@@ -307,20 +307,27 @@ public class AppSchedulable extends Sche
// (not scheduled) in order to promote better locality.
synchronized (app) {
for (Priority priority : prioritiesToTry) {
+ if (app.getTotalRequiredResources(priority) <= 0) {
+ continue;
+ }
+
app.addSchedulingOpportunity(priority);
- NodeType allowedLocality = app.getAllowedLocalityLevel(priority,
- scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(),
- scheduler.getRackLocalityThreshold());
+ ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
+ node.getRackName());
ResourceRequest localRequest = app.getResourceRequest(priority,
node.getHostName());
- if (localRequest != null && localRequest.getNumContainers() != 0) {
+
+ NodeType allowedLocality = app.getAllowedLocalityLevel(priority,
+ scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(),
+ scheduler.getRackLocalityThreshold());
+
+ if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
+ && localRequest != null && localRequest.getNumContainers() != 0) {
return assignContainer(node, app, priority,
localRequest, NodeType.NODE_LOCAL, reserved);
}
- ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
- node.getRackName());
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
&& (allowedLocality.equals(NodeType.RACK_LOCAL) ||
allowedLocality.equals(NodeType.OFF_SWITCH))) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1433484&r1=1433483&r2=1433484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Jan 15 16:12:33 2013
@@ -1275,4 +1275,46 @@ public class TestFairScheduler {
FSSchedulerApp app2 = scheduler.applications.get(attId2);
assertNull("The application was allowed", app2);
}
+
+ @Test
+ public void testMultipleNodesSingleRackRequest() throws Exception {
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ RMNode node3 = MockNodes.newNodeInfo(2, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
+ scheduler.addApplication(appId, "queue1", "user1");
+
+ // 1 request with 2 nodes on the same rack. another request with 1 node on
+ // a different rack
+ List<ResourceRequest> asks = new ArrayList<ResourceRequest>();
+ asks.add(createResourceRequest(1024, node1.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node2.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1));
+ asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1));
+ asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2));
+
+ scheduler.allocate(appId, asks, new ArrayList<ContainerId>());
+
+ // node 1 checks in
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+ scheduler.handle(updateEvent1);
+ // should assign node local
+ assertEquals(1, scheduler.applications.get(appId).getLiveContainers().size());
+
+ // node 2 checks in
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+ scheduler.handle(updateEvent2);
+ // should assign rack local
+ assertEquals(2, scheduler.applications.get(appId).getLiveContainers().size());
+ }
}