You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2018/09/10 20:00:16 UTC
hadoop git commit: YARN-8709: CS preemption monitor always fails
since one under-served queue was deleted. Contributed by Tao Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 317f317d4 -> 987d8191a
YARN-8709: CS preemption monitor always fails since one under-served queue was deleted. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/987d8191
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/987d8191
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/987d8191
Branch: refs/heads/trunk
Commit: 987d8191ad409298570f7ef981e9bc8fb72ff16c
Parents: 317f317
Author: Eric E Payne <er...@oath.com>
Authored: Mon Sep 10 19:55:20 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Mon Sep 10 19:55:20 2018 +0000
----------------------------------------------------------------------
.../ProportionalCapacityPreemptionPolicy.java | 3 +
...alCapacityPreemptionPolicyMockFramework.java | 5 ++
...ionalCapacityPreemptionPolicyIntraQueue.java | 90 +++++++++++++++++++-
3 files changed, 94 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/987d8191/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 036fd2f..2156b09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -467,6 +467,9 @@ public class ProportionalCapacityPreemptionPolicy
Resource totalPreemptionAllowed = Resources.multiply(clusterResources,
percentageClusterPreemptionAllowed);
+ //clear under served queues for every run
+ partitionToUnderServedQueues.clear();
+
// based on ideal allocation select containers to be preemptionCandidates from each
// queue and each application
Map<ApplicationAttemptId, Set<RMContainer>> toPreempt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/987d8191/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 64b56fb..fa66cbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -206,6 +206,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
mClock);
}
+ public void updateQueueConfig(String queuesConfig) {
+ ParentQueue root = mockQueueHierarchy(queuesConfig);
+ when(cs.getRootQueue()).thenReturn(root);
+ }
+
private void mockContainers(String containersConfig, FiCaSchedulerApp app,
ApplicationAttemptId attemptId, String queueName,
List<RMContainer> reservedContainers, List<RMContainer> liveContainers) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/987d8191/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
index 6c5aa67..750054c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
@@ -71,9 +71,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueue
"n1= res=100";
String queuesConfig =
// guaranteed,max,used,pending,reserved
- "root(=[100 100 79 120 0]);" + // root
+ "root(=[100 100 79 110 0]);" + // root
"-a(=[11 100 11 50 0]);" + // a
- "-b(=[40 100 38 60 0]);" + // b
+ "-b(=[40 100 38 50 0]);" + // b
"-c(=[20 100 10 10 0]);" + // c
"-d(=[29 100 20 0 0])"; // d
@@ -128,9 +128,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueue
"n1= res=100";
String queuesConfig =
// guaranteed,max,used,pending,reserved
- "root(=[100 100 80 120 0]);" + // root
+ "root(=[100 100 80 110 0]);" + // root
"-a(=[11 100 11 50 0]);" + // a
- "-b(=[40 100 38 60 0]);" + // b
+ "-b(=[40 100 38 50 0]);" + // b
"-c(=[20 100 10 10 0]);" + // c
"-d(=[29 100 20 0 0])"; // d
@@ -942,4 +942,86 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueue
verify(mDisp, times(22))
.handle(argThat(new IsPreemptionRequestFor(getAppAttemptId(6))));
}
+
+ @Test
+ public void testIntraQueuePreemptionAfterQueueDropped()
+ throws IOException {
+ /**
+ * Test intra queue preemption after under-served queue dropped,
+ * At first, Queue structure is:
+ *
+ * <pre>
+ * root
+ * / | | \
+ * a b c d
+ * </pre>
+ *
+ * After dropped under-served queue "c", Queue structure is:
+ *
+ * <pre>
+ * root
+ * / | \
+ * a b d
+ * </pre>
+ *
+ * Verify no exception is thrown and preemption results is correct
+ */
+ conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
+ "priority_first");
+
+ String labelsConfig = "=100,true;";
+ String nodesConfig = // n1 has no label
+ "n1= res=100";
+ String queuesConfig =
+ // guaranteed,max,used,pending,reserved
+ "root(=[100 100 79 110 0]);" + // root
+ "-a(=[11 100 11 50 0]);" + // a
+ "-b(=[40 100 38 50 0]);" + // b
+ "-c(=[20 100 10 10 0]);" + // c
+ "-d(=[29 100 20 0 0])"; // d
+
+ String appsConfig =
+ // queueName\t(priority,resource,host,expression,#repeat,reserved,
+ // pending)
+ "a\t" // app1 in a
+ + "(1,1,n1,,6,false,25);" + // app1 a
+ "a\t" // app2 in a
+ + "(1,1,n1,,5,false,25);" + // app2 a
+ "b\t" // app3 in b
+ + "(4,1,n1,,34,false,20);" + // app3 b
+ "b\t" // app4 in b
+ + "(4,1,n1,,2,false,10);" + // app4 b
+ "b\t" // app4 in b
+ + "(5,1,n1,,1,false,10);" + // app5 b
+ "b\t" // app4 in b
+ + "(6,1,n1,,1,false,10);" + // app6 in b
+ "c\t" // app1 in a
+ + "(1,1,n1,,10,false,10);" + "d\t" // app7 in c
+ + "(1,1,n1,,20,false,0)";
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ queuesConfig =
+ // guaranteed,max,used,pending,reserved
+ "root(=[100 100 69 100 0]);" + // root
+ "-a(=[11 100 11 50 0]);" + // a
+ "-b(=[40 100 38 50 0]);" + // b
+ "-d(=[49 100 20 0 0])"; // d
+
+ updateQueueConfig(queuesConfig);
+
+ // will throw YarnRuntimeException(This shouldn't happen, cannot find
+ // TempQueuePerPartition for queueName=c) without patch in YARN-8709
+ policy.editSchedule();
+
+ // For queue B, app3 and app4 were of lower priority. Hence take 8
+ // containers from them by hitting the intraQueuePreemptionDemand of 20%.
+ verify(mDisp, times(1)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(4))));
+ verify(mDisp, times(7)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(3))));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org