You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/20 18:42:13 UTC
[05/17] hadoop git commit: YARN-3885.
ProportionalCapacityPreemptionPolicy doesn't preempt if queue is more than 2
level. (Ajith S via wangda)
YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue is more than 2 level. (Ajith S via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3540d5fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3540d5fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3540d5fe
Branch: refs/heads/HADOOP-12111
Commit: 3540d5fe4b1da942ea80c9e7ca1126b1abb8a68a
Parents: fa2b63e
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jul 16 16:13:32 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jul 16 16:13:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../ProportionalCapacityPreemptionPolicy.java | 8 ++++-
...estProportionalCapacityPreemptionPolicy.java | 32 ++++++++++++++++++++
3 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3540d5fe/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cd0d132..9a6f4d2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -640,6 +640,9 @@ Release 2.8.0 - UNRELEASED
YARN-3930. FileSystemNodeLabelsStore should make sure edit log file closed when
exception is thrown. (Dian Fu via wangda)
+ YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue is
+ more than 2 level. (Ajith S via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3540d5fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 6e661d4..1152cef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -896,8 +896,10 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
ret.untouchableExtra = Resource.newInstance(0, 0);
} else {
ret.untouchableExtra =
- Resources.subtractFrom(extra, childrensPreemptable);
+ Resources.subtract(extra, childrensPreemptable);
}
+ ret.preemptableExtra = Resources.min(
+ rc, partitionResource, childrensPreemptable, extra);
}
}
addTempQueuePartition(ret);
@@ -1127,4 +1129,8 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic
}
}
+ @VisibleForTesting
+ public Map<String, Map<String, TempQueuePerPartition>> getQueuePartitions() {
+ return queueToPartitions;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3540d5fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 3057360..bc4d0dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.TempQueuePerPartition;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -898,6 +899,37 @@ public class TestProportionalCapacityPreemptionPolicy {
verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appA)));
}
+ @Test
+ public void testHierarchicalLarge3Levels() {
+ int[][] qData = new int[][] {
+ // / A F I
+ // B C G H J K
+ // D E
+ { 400, 200, 60, 140, 100, 40, 100, 70, 30, 100, 10, 90 }, // abs
+ { 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, 400 }, // maxCap
+ { 400, 210, 60, 150, 100, 50, 100, 50, 50, 90, 10, 80 }, // used
+ { 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 }, // pending
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // reserved
+ // appA appB appC appD appE appF appG
+ { 7, 3, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1 }, // apps
+ { -1, -1, 1, -1, 1, 1, -1, 1, 1, -1, 1, 1 }, // req granularity
+ { 3, 2, 0, 2, 0, 0, 2, 0, 0, 2, 0, 0 }, // subqueues
+ };
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+ // XXX note: compensating for rounding error in Resources.multiplyTo
+ // which is likely triggered since we use small numbers for readability
+ //run with Logger.getRootLogger().setLevel(Level.DEBUG);
+ verify(mDisp, times(9)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ assertEquals(10, policy.getQueuePartitions().get("queueE").get("").preemptableExtra.getMemory());
+ //2nd level child(E) preempts 10, but parent A has only 9 extra
+ //check the parent can prempt only the extra from > 2 level child
+ TempQueuePerPartition tempQueueAPartition = policy.getQueuePartitions().get("queueA").get("");
+ assertEquals(0, tempQueueAPartition.untouchableExtra.getMemory());
+ int extraForQueueA = tempQueueAPartition.current.getMemory()- tempQueueAPartition.guaranteed.getMemory();
+ assertEquals(extraForQueueA,tempQueueAPartition.preemptableExtra.getMemory());
+ }
+
static class IsPreemptionRequestFor
extends ArgumentMatcher<ContainerPreemptEvent> {
private final ApplicationAttemptId appAttId;