You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/02/27 15:15:23 UTC
[27/31] hadoop git commit: YARN-6172. FSLeafQueue demand update needs
to be atomic. (Miklos Szegedi via kasha)
YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de
Branch: refs/heads/HADOOP-13345
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Sun Feb 26 20:36:33 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 21 +++++------------
.../scheduler/fair/TestFSAppStarvation.java | 24 ++++++++++++++++----
2 files changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- demand = Resources.createResource(0);
+ Resource tmpDemand = Resources.createResource(0);
readLock.lock();
try {
for (FSAppAttempt sched : runnableApps) {
- updateDemandForApp(sched);
+ sched.updateDemand();
+ Resources.addTo(tmpDemand, sched.getDemand());
}
for (FSAppAttempt sched : nonRunnableApps) {
- updateDemandForApp(sched);
+ sched.updateDemand();
+ Resources.addTo(tmpDemand, sched.getDemand());
}
} finally {
readLock.unlock();
}
// Cap demand to maxShare to limit allocation to maxShare
- demand = Resources.componentwiseMin(demand, maxShare);
+ demand = Resources.componentwiseMin(tmpDemand, maxShare);
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
+ "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
+ getFairShare());
}
}
-
- private void updateDemandForApp(FSAppAttempt sched) {
- sched.updateDemand();
- Resource toAdd = sched.getDemand();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
- + "; Total resource demand for " + getName() + " now "
- + demand);
- }
- demand = Resources.add(demand, toAdd);
- }
@Override
public Resource assignContainer(FSSchedulerNode node) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
public void testPreemptionEnabled() throws Exception {
setupClusterAndSubmitJobs();
+ // Wait for apps to be processed by MockPreemptionThread
+ for (int i = 0; i < 6000; ++i) {
+ if (preemptionThread.uniqueAppsAdded() >= 3) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+
assertNotNull("FSContext does not have an FSStarvedApps instance",
scheduler.getContext().getStarvedApps());
assertEquals("Expecting 3 starved applications, one each for the "
@@ -113,8 +121,19 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
clock.tickMsec(
FairSchedulerWithMockPreemption.DELAY_FOR_NEXT_STARVATION_CHECK_MS);
scheduler.update();
+
+ // Wait for apps to be processed by MockPreemptionThread
+ for (int i = 0; i < 6000; ++i) {
+ if(preemptionThread.totalAppsAdded() >
+ preemptionThread.uniqueAppsAdded()) {
+ break;
+ }
+ Thread.sleep(10);
+ }
+
assertTrue("Each app is marked as starved exactly once",
- preemptionThread.totalAppsAdded() > preemptionThread.uniqueAppsAdded());
+ preemptionThread.totalAppsAdded() >
+ preemptionThread.uniqueAppsAdded());
}
/*
@@ -154,9 +173,6 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
// Scheduler update to populate starved apps
scheduler.update();
-
- // Wait for apps to be processed by MockPreemptionThread
- Thread.yield();
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org