You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/12/08 18:59:15 UTC
hadoop git commit: YARN-7469. Capacity Scheduler Intra-queue
preemption: User can starve if newest app is exactly at user limit.
Contributed by Eric Payne.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0.0 b1712f9c1 -> 0510ceede
YARN-7469. Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit. Contributed by Eric Payne.
(cherry picked from commit 61ace174cdcbca9d22abce7aa0aa71148f37ad55)
(cherry picked from commit c3fb49667a4c11d993056e9e3c8ca4ec9479538f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0510ceed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0510ceed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0510ceed
Branch: refs/heads/branch-3.0.0
Commit: 0510ceede63cc6a7b35a866a6e33d0ff9da1e3af
Parents: b1712f9
Author: Sunil G <su...@apache.org>
Authored: Thu Nov 16 22:34:23 2017 +0530
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Dec 8 10:59:09 2017 -0800
----------------------------------------------------------------------
.../FifoIntraQueuePreemptionPlugin.java | 6 ++++
...alCapacityPreemptionPolicyMockFramework.java | 3 ++
...cityPreemptionPolicyIntraQueueUserLimit.java | 35 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0510ceed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index 00ae3da..3332f2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -203,6 +203,12 @@ public class FifoIntraQueuePreemptionPlugin
Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.selected);
Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.getAMUsed());
+ if (context.getIntraQueuePreemptionOrderPolicy()
+ .equals(IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) {
+ Resources.subtractFromNonNegative(preemtableFromApp,
+ tmpApp.getFiCaSchedulerApp().getCSLeafQueue().getMinimumAllocation());
+ }
+
// Calculate toBePreempted from apps as follows:
// app.preemptable = min(max(app.used - app.selected - app.ideal, 0),
// intra_q_preemptable)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0510ceed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 4fc0ea4..0bc5cb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -358,6 +358,9 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
queue = (LeafQueue) nameToCSQueues.get(queueName);
queue.getApplications().add(app);
queue.getAllApplications().add(app);
+ when(queue.getMinimumAllocation())
+ .thenReturn(Resource.newInstance(1,1));
+ when(app.getCSLeafQueue()).thenReturn(queue);
HashSet<String> users = userMap.get(queueName);
if (null == users) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0510ceed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
index 7df52f9..0440db3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
@@ -896,4 +896,39 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
+
+ @Test
+ public void testSimpleIntraQueuePreemptionOneUserUnderOneUserAtOneUserAbove()
+ throws IOException {
+ conf.setFloat(CapacitySchedulerConfiguration.
+ INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT,
+ (float) 0.5);
+
+ String labelsConfig = "=100,true;";
+ String nodesConfig = // n1 has no label
+ "n1= res=100";
+ String queuesConfig =
+ // guaranteed,max,used,pending,reserved
+ "root(=[100 100 100 1 0]);" + // root
+ "-a(=[100 100 100 1 0])"; // a
+
+ String appsConfig =
+ // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+ "a\t" // app1 in a
+ + "(1,1,n1,,65,false,0,user1);" +
+ "a\t" // app2 in a
+ + "(1,1,n1,,35,false,0,user2);" +
+ "a\t" // app3 in a
+ + "(1,1,n1,,0,false,1,user3)"
+ ;
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ // app2 is right at its user limit and app1 needs one resource. Should
+ // preempt 1 container.
+ verify(mDisp, times(1)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(1))));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org