You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/05/02 19:21:38 UTC
[03/50] [abbrv] hadoop git commit: YARN-7527. Over-allocate node
resource in async-scheduling mode of CapacityScheduler. Contributed by Tao
Yang.
YARN-7527. Over-allocate node resource in async-scheduling mode of CapacityScheduler. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a48deb15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a48deb15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a48deb15
Branch: refs/heads/YARN-8200
Commit: a48deb1552c8b920316a630daf9748f821f741af
Parents: be627cc
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Apr 12 10:12:46 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Apr 12 10:12:46 2018 +0800
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 6 +-
.../TestCapacitySchedulerAsyncScheduling.java | 74 ++++++++++++++++++++
2 files changed, 78 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48deb15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 726f7e2..cab4dd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -424,8 +424,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Common part of check container allocation regardless if it is a
// increase container or regular container
- commonCheckContainerAllocation(cluster, allocation,
- schedulerContainer);
+ if (!commonCheckContainerAllocation(cluster, allocation,
+ schedulerContainer)) {
+ return false;
+ }
} else {
// Container reserved first time will be NEW, after the container
// accepted & confirmed, it will become RESERVED state
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a48deb15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index da06557..7e092e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -411,6 +411,80 @@ public class TestCapacitySchedulerAsyncScheduling {
rm.stop();
}
+ @Test (timeout = 30000)
+ public void testNodeResourceOverAllocated()
+ throws Exception {
+ // disable async-scheduling for simulating complex scene
+ Configuration disableAsyncConf = new Configuration(conf);
+ disableAsyncConf.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+ // init RM & NMs & Nodes
+ final MockRM rm = new MockRM(disableAsyncConf);
+ rm.start();
+ final MockNM nm1 = rm.registerNode("h1:1234", 9 * GB);
+ final MockNM nm2 = rm.registerNode("h2:1234", 9 * GB);
+ List<MockNM> nmLst = new ArrayList<>();
+ nmLst.add(nm1);
+ nmLst.add(nm2);
+
+ // init scheduler & nodes
+ while (
+ ((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+ .nodeCount() < 2) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals(2,
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getNodeTracker().nodeCount());
+ CapacityScheduler scheduler =
+ (CapacityScheduler) rm.getRMContext().getScheduler();
+ SchedulerNode sn1 = scheduler.getSchedulerNode(nm1.getNodeId());
+
+ // launch app
+ RMApp app = rm.submitApp(200, "app", "user", null, false, "default",
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ FiCaSchedulerApp schedulerApp =
+ scheduler.getApplicationAttempt(am.getApplicationAttemptId());
+ // allocate 2 containers and running on nm1
+ Resource containerResource = Resources.createResource(5 * GB);
+ am.allocate(Arrays.asList(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*", containerResource, 2)),
+ null);
+
+ // generate over-allocated proposals for nm1
+ for (int containerNo = 2; containerNo <= 3; containerNo++) {
+ Container container = Container.newInstance(
+ ContainerId.newContainerId(am.getApplicationAttemptId(), containerNo),
+ sn1.getNodeID(), sn1.getHttpAddress(), containerResource,
+ Priority.newInstance(0), null);
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.create(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*", containerResource, 1)),
+ am.getApplicationAttemptId(), sn1.getNodeID(), "user",
+ rm.getRMContext());
+ SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> newContainer =
+ new SchedulerContainer<>(schedulerApp,
+ scheduler.getNode(sn1.getNodeID()), rmContainer, "", true);
+ ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>
+ newContainerProposal =
+ new ContainerAllocationProposal<>(newContainer, null, null,
+ NodeType.OFF_SWITCH, NodeType.OFF_SWITCH,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, containerResource);
+ List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>>
+ newProposals = new ArrayList<>();
+ newProposals.add(newContainerProposal);
+ ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request =
+ new ResourceCommitRequest<>(newProposals, null, null);
+ scheduler.tryCommit(scheduler.getClusterResource(), request);
+ }
+ // make sure node resource can't be over-allocated!
+ Assert.assertTrue("Node resource is Over-allocated!",
+ sn1.getUnallocatedResource().getMemorySize() > 0);
+ rm.stop();
+ }
+
private void allocateAndLaunchContainers(MockAM am, MockNM nm, MockRM rm,
int nContainer, Resource resource, int priority, int startContainerId)
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org