You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/02/24 19:34:33 UTC
[06/50] [abbrv] hadoop git commit: YARN-6194. Cluster capacity in
SchedulingPolicy is updated only on allocation file reload. (Yufei Gu via
kasha)
YARN-6194. Cluster capacity in SchedulingPolicy is updated only on allocation file reload. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b10e9622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b10e9622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b10e9622
Branch: refs/heads/YARN-2915
Commit: b10e962224a8ae1c6031a05322b0cc5e564bd078
Parents: 718ad9f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Wed Feb 22 15:58:49 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Wed Feb 22 15:58:49 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSContext.java | 21 ++++++++++++----
.../resourcemanager/scheduler/fair/FSQueue.java | 2 +-
.../scheduler/fair/FairScheduler.java | 6 ++---
.../scheduler/fair/SchedulingPolicy.java | 19 ++++++++++++++-
.../DominantResourceFairnessPolicy.java | 16 +++++++------
.../scheduler/fair/TestFairScheduler.java | 8 +++----
.../TestDominantResourceFairnessPolicy.java | 25 +++++++++++++++++++-
7 files changed, 74 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
index 56bc99c..a4aa8f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+import org.apache.hadoop.yarn.api.records.Resource;
+
/**
* Helper class that holds basic information to be passed around
* FairScheduler classes. Think of this as a glorified map that holds key
@@ -27,28 +29,37 @@ public class FSContext {
private boolean preemptionEnabled = false;
private float preemptionUtilizationThreshold;
private FSStarvedApps starvedApps;
+ private FairScheduler scheduler;
+
+ FSContext(FairScheduler scheduler) {
+ this.scheduler = scheduler;
+ }
- public boolean isPreemptionEnabled() {
+ boolean isPreemptionEnabled() {
return preemptionEnabled;
}
- public void setPreemptionEnabled() {
+ void setPreemptionEnabled() {
this.preemptionEnabled = true;
if (starvedApps == null) {
starvedApps = new FSStarvedApps();
}
}
- public FSStarvedApps getStarvedApps() {
+ FSStarvedApps getStarvedApps() {
return starvedApps;
}
- public float getPreemptionUtilizationThreshold() {
+ float getPreemptionUtilizationThreshold() {
return preemptionUtilizationThreshold;
}
- public void setPreemptionUtilizationThreshold(
+ void setPreemptionUtilizationThreshold(
float preemptionUtilizationThreshold) {
this.preemptionUtilizationThreshold = preemptionUtilizationThreshold;
}
+
+ public Resource getClusterResource() {
+ return scheduler.getClusterResource();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 7e8b858..b5592c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -135,7 +135,7 @@ public abstract class FSQueue implements Queue, Schedulable {
}
public void setPolicy(SchedulingPolicy policy) {
- policy.initialize(scheduler.getClusterResource());
+ policy.initialize(scheduler.getContext());
this.policy = policy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a15e6b5..c946bfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -205,13 +205,12 @@ public class FairScheduler extends
public FairScheduler() {
super(FairScheduler.class.getName());
- context = new FSContext();
+ context = new FSContext(this);
allocsLoader = new AllocationFileLoaderService();
queueMgr = new QueueManager(this);
maxRunningEnforcer = new MaxRunningAppsEnforcer(this);
}
- @VisibleForTesting
public FSContext getContext() {
return context;
}
@@ -1452,8 +1451,7 @@ public class FairScheduler extends
} else {
allocConf = queueInfo;
setQueueAcls(allocConf.getQueueAcls());
- allocConf.getDefaultSchedulingPolicy().initialize(
- getClusterResource());
+ allocConf.getDefaultSchedulingPolicy().initialize(getContext());
queueMgr.updateAllocationConfiguration(allocConf);
applyChildDefaults();
maxRunningEnforcer.updateRunnabilityOnReload();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index 3fe36f3..9a9be8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -91,10 +91,27 @@ public abstract class SchedulingPolicy {
}
return getInstance(clazz);
}
-
+
+ /**
+ * Initialize the scheduling policy with cluster resources.
+ * @deprecated Since it doesn't track cluster resource changes, replaced by
+ * {@link #initialize(FSContext)}.
+ *
+ * @param clusterCapacity cluster resources
+ */
+ @Deprecated
public void initialize(Resource clusterCapacity) {}
/**
+ * Initialize the scheduling policy with a {@link FSContext} object, which has
+ * a pointer to the cluster resources among other information.
+ *
+ * @param fsContext a {@link FSContext} object which has a pointer to the
+ * cluster resources
+ */
+ public void initialize(FSContext fsContext) {}
+
+ /**
* The {@link ResourceCalculator} returned by this method should be used
* for any calculations involving resources.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 369b8a1..193ed4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
@@ -104,17 +105,17 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
}
@Override
- public void initialize(Resource clusterCapacity) {
- COMPARATOR.setClusterCapacity(clusterCapacity);
+ public void initialize(FSContext fsContext) {
+ COMPARATOR.setFSContext(fsContext);
}
public static class DominantResourceFairnessComparator implements Comparator<Schedulable> {
private static final int NUM_RESOURCES = ResourceType.values().length;
-
- private Resource clusterCapacity;
- public void setClusterCapacity(Resource clusterCapacity) {
- this.clusterCapacity = clusterCapacity;
+ private FSContext fsContext;
+
+ public void setFSContext(FSContext fsContext) {
+ this.fsContext = fsContext;
}
@Override
@@ -125,7 +126,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
ResourceWeights sharesOfMinShare2 = new ResourceWeights();
ResourceType[] resourceOrder1 = new ResourceType[NUM_RESOURCES];
ResourceType[] resourceOrder2 = new ResourceType[NUM_RESOURCES];
-
+ Resource clusterCapacity = fsContext.getClusterResource();
+
// Calculate shares of the cluster for each resource both schedulables.
calculateShares(s1.getResourceUsage(),
clusterCapacity, sharesOfCluster1, resourceOrder1, s1.getWeights());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 4def53f..62430bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3293,7 +3293,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.update();
@@ -3339,7 +3339,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.update();
@@ -3354,7 +3354,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
Assert.assertEquals(1, app2.getLiveContainers().size());
}
-
+
@Test
public void testDRFHierarchicalQueues() throws Exception {
scheduler.init(conf);
@@ -3384,7 +3384,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app4 = scheduler.getSchedulerApp(appAttId4);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1.subqueue1").setPolicy(drfPolicy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
index a5c20c1..3719e2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
@@ -19,12 +19,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.Comparator;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -40,7 +43,10 @@ public class TestDominantResourceFairnessPolicy {
private Comparator<Schedulable> createComparator(int clusterMem,
int clusterCpu) {
DominantResourceFairnessPolicy policy = new DominantResourceFairnessPolicy();
- policy.initialize(BuilderUtils.newResource(clusterMem, clusterCpu));
+ FSContext fsContext = mock(FSContext.class);
+ when(fsContext.getClusterResource()).
+ thenReturn(Resources.createResource(clusterMem, clusterCpu));
+ policy.initialize(fsContext);
return policy.getComparator();
}
@@ -160,4 +166,21 @@ public class TestDominantResourceFairnessPolicy {
assertEquals(ResourceType.CPU, resourceOrder[0]);
assertEquals(ResourceType.MEMORY, resourceOrder[1]);
}
+
+ @Test
+ public void testCompareSchedulablesWithClusterResourceChanges(){
+ Schedulable schedulable1 = createSchedulable(2000, 1);
+ Schedulable schedulable2 = createSchedulable(1000, 2);
+
+ // schedulable1 has share weights [1/2, 1/5], schedulable2 has share
+ // weights [1/4, 2/5], schedulable1 > schedulable2 since 1/2 > 2/5
+ assertTrue(createComparator(4000, 5)
+ .compare(schedulable1, schedulable2) > 0);
+
+ // share weights have changed because of the cluster resource change.
+ // schedulable1 has share weights [1/4, 1/6], schedulable2 has share
+ // weights [1/8, 1/3], schedulable1 < schedulable2 since 1/4 < 1/3
+ assertTrue(createComparator(8000, 6)
+ .compare(schedulable1, schedulable2) < 0);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org