You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/11/16 13:46:13 UTC
[02/50] [abbrv] hadoop git commit: YARN-5453. FairScheduler#update
may skip update demand resource of child queue/app if current demand reached
maxResource. (sandflee via kasha)
YARN-5453. FairScheduler#update may skip update demand resource of child queue/app if current demand reached maxResource. (sandflee via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86ac1ad9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86ac1ad9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86ac1ad9
Branch: refs/heads/YARN-3926
Commit: 86ac1ad9fd65c7dd12278372b369de38dc4616db
Parents: c8bc7a8
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Nov 9 23:44:02 2016 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Nov 9 23:44:02 2016 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 15 +++----
.../scheduler/fair/FSParentQueue.java | 6 +--
.../scheduler/fair/TestFairScheduler.java | 41 ++++++++++++++++++++
3 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 9d5bbe5..c393759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -268,20 +268,16 @@ public class FSLeafQueue extends FSQueue {
readLock.lock();
try {
for (FSAppAttempt sched : runnableApps) {
- if (Resources.equals(demand, maxShare)) {
- break;
- }
- updateDemandForApp(sched, maxShare);
+ updateDemandForApp(sched);
}
for (FSAppAttempt sched : nonRunnableApps) {
- if (Resources.equals(demand, maxShare)) {
- break;
- }
- updateDemandForApp(sched, maxShare);
+ updateDemandForApp(sched);
}
} finally {
readLock.unlock();
}
+ // Cap demand to maxShare to limit allocation to maxShare
+ demand = Resources.componentwiseMin(demand, maxShare);
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
+ "; the max is " + maxShare);
@@ -290,7 +286,7 @@ public class FSLeafQueue extends FSQueue {
}
}
- private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) {
+ private void updateDemandForApp(FSAppAttempt sched) {
sched.updateDemand();
Resource toAdd = sched.getDemand();
if (LOG.isDebugEnabled()) {
@@ -299,7 +295,6 @@ public class FSLeafQueue extends FSQueue {
+ demand);
}
demand = Resources.add(demand, toAdd);
- demand = Resources.componentwiseMin(demand, maxRes);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index d05390b..53ac8c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -159,16 +159,14 @@ public class FSParentQueue extends FSQueue {
childQueue.updateDemand();
Resource toAdd = childQueue.getDemand();
demand = Resources.add(demand, toAdd);
- demand = Resources.componentwiseMin(demand, maxShare);
if (LOG.isDebugEnabled()) {
LOG.debug("Counting resource from " + childQueue.getName() + " " +
toAdd + "; Total resource demand for " + getName() +
" now " + demand);
}
- if (Resources.equals(demand, maxShare)) {
- break;
- }
}
+ // Cap demand to maxShare to limit allocation to maxShare
+ demand = Resources.componentwiseMin(demand, maxShare);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index f17726c..21d22c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4813,4 +4813,45 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(0, metrics.getReservedMB());
assertEquals(0, metrics.getReservedVirtualCores());
}
+
+
+ @Test
+ public void testUpdateDemand() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ Resource maxResource = Resources.createResource(1024 * 8);
+
+ FSAppAttempt app1 = mock(FSAppAttempt.class);
+ Mockito.when(app1.getDemand()).thenReturn(maxResource);
+ FSAppAttempt app2 = mock(FSAppAttempt.class);
+ Mockito.when(app2.getDemand()).thenReturn(maxResource);
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+
+ FSLeafQueue aQueue =
+ new FSLeafQueue("root.queue1.a", scheduler, queue1);
+ aQueue.setMaxShare(maxResource);
+ aQueue.addAppSchedulable(app1);
+
+ FSLeafQueue bQueue =
+ new FSLeafQueue("root.queue1.b", scheduler, queue1);
+ bQueue.setMaxShare(maxResource);
+ bQueue.addAppSchedulable(app2);
+
+ queue1.setMaxShare(maxResource);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ queue1.updateDemand();
+
+ assertTrue("Demand is greater than max allowed ",
+ Resources.equals(queue1.getDemand(), maxResource));
+ assertTrue("Demand of child queue not updated ",
+ Resources.equals(aQueue.getDemand(), maxResource) &&
+ Resources.equals(bQueue.getDemand(), maxResource));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org