You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/06/07 20:26:11 UTC
[19/50] [abbrv] hadoop git commit: YARN-5165. Fix NoOvercommitPolicy
to take advantage of RLE representation of plan. (Carlo Curino via asuresh)
YARN-5165. Fix NoOvercommitPolicy to take advantage of RLE representation of plan. (Carlo Curino via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db54670e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db54670e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db54670e
Branch: refs/heads/YARN-4757
Commit: db54670e83a84c1d7deff2c225725687cf9e5f14
Parents: f10ebc6
Author: Arun Suresh <as...@apache.org>
Authored: Fri Jun 3 14:49:32 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Jun 3 14:49:32 2016 -0700
----------------------------------------------------------------------
.../reservation/NoOverCommitPolicy.java | 38 ++++++++------------
.../planning/TestSimpleCapacityReplanner.java | 2 +-
2 files changed, 15 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db54670e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
index 119520b..814d4b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -21,11 +21,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
-import org.apache.hadoop.yarn.util.resource.Resources;
/**
* This policy enforce a simple physical cluster capacity constraints, by
@@ -52,29 +50,21 @@ public class NoOverCommitPolicy implements SharingPolicy {
+ oldReservation.getUser() + " != " + reservation.getUser());
}
- long startTime = reservation.getStartTime();
- long endTime = reservation.getEndTime();
- long step = plan.getStep();
+ RLESparseResourceAllocation available = plan.getAvailableResourceOverTime(
+ reservation.getUser(), reservation.getReservationId(),
+ reservation.getStartTime(), reservation.getEndTime());
- // for every instant in time, check we are respecting cluster capacity
- for (long t = startTime; t < endTime; t += step) {
- Resource currExistingAllocTot = plan.getTotalCommittedResources(t);
- Resource currNewAlloc = reservation.getResourcesAtTime(t);
- Resource currOldAlloc = Resource.newInstance(0, 0);
- if (oldReservation != null) {
- oldReservation.getResourcesAtTime(t);
- }
- // check the cluster is never over committed
- // currExistingAllocTot + currNewAlloc - currOldAlloc >
- // capPlan.getTotalCapacity()
- if (Resources.greaterThan(plan.getResourceCalculator(), plan
- .getTotalCapacity(), Resources.subtract(
- Resources.add(currExistingAllocTot, currNewAlloc), currOldAlloc),
- plan.getTotalCapacity())) {
- throw new ResourceOverCommitException("Resources at time " + t
- + " would be overcommitted by " + "accepting reservation: "
- + reservation.getReservationId());
- }
+ // test the reservation does not exceed what is available
+ try {
+ RLESparseResourceAllocation
+ .merge(plan.getResourceCalculator(), plan.getTotalCapacity(),
+ available, reservation.getResourcesOverTime(),
+ RLESparseResourceAllocation.RLEOperator.subtractTestNonNegative,
+ reservation.getStartTime(), reservation.getEndTime());
+ } catch (PlanningException p) {
+ throw new ResourceOverCommitException(
+ "Resources at time " + " would be overcommitted by "
+ + "accepting reservation: " + reservation.getReservationId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db54670e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
index d762b36..e01608c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
@@ -53,7 +53,7 @@ public class TestSimpleCapacityReplanner {
@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
- Resource clusterCapacity = Resource.newInstance(100 * 1024, 10);
+ Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
Resource minAlloc = Resource.newInstance(1024, 1);
Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org