You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2018/09/19 11:54:07 UTC
hadoop git commit: Revert "YARN-8771. CapacityScheduler fails to
unreserve when cluster resource contains empty resource type. Contributed by
Tao Yang."
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 f56e36935 -> 4edcde055
Revert "YARN-8771. CapacityScheduler fails to unreserve when cluster resource contains empty resource type. Contributed by Tao Yang."
This reverts commit f56e36935da69b5b12086d80bfe1cdb446213882.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4edcde05
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4edcde05
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4edcde05
Branch: refs/heads/branch-3.0
Commit: 4edcde055b34e24946cdb4002bef120f69599eb3
Parents: f56e369
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Sep 19 19:47:06 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Sep 19 19:47:06 2018 +0800
----------------------------------------------------------------------
.../allocator/RegularContainerAllocator.java | 3 +-
.../capacity/TestContainerAllocation.java | 68 --------------------
2 files changed, 2 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4edcde05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index c6b3ca9..69e90c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -530,7 +530,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
currentResoureLimits.getAmountNeededUnreserve());
boolean needToUnreserve =
- rc.isAnyMajorResourceAboveZero(resourceNeedToUnReserve);
+ Resources.greaterThan(rc, clusterResource,
+ resourceNeedToUnReserve, Resources.none());
RMContainer unreservedContainer = null;
boolean reservationsContinueLooking =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4edcde05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index f373f15..48a0f3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -59,10 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -986,68 +982,4 @@ public class TestContainerAllocation {
Assert.assertEquals(2, lq.getMetrics().getAppsPending());
rm1.close();
}
-
- @Test(timeout = 60000)
- public void testUnreserveWhenClusterResourceHasEmptyResourceType()
- throws Exception {
- /**
- * Test case:
- * Create a cluster with two nodes whose node resource both are
- * <8GB, 8core, 0>, create queue "a" whose max-resource is <8GB, 8 core, 0>,
- * submit app1 to queue "a" whose am use <1GB, 1 core, 0> and launch on nm1,
- * submit app2 to queue "b" whose am use <1GB, 1 core, 0> and launch on nm1,
- * app1 asks two <7GB, 1core> containers and nm1 do 1 heartbeat,
- * then scheduler reserves one container on nm1.
- *
- * After nm2 do next node heartbeat, scheduler should unreserve the reserved
- * container on nm1 then allocate a container on nm2.
- */
- TestResourceUtils.addNewTypesToResources("resource1");
- CapacitySchedulerConfiguration newConf =
- (CapacitySchedulerConfiguration) TestUtils
- .getConfigurationWithMultipleQueues(conf);
- newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
- DominantResourceCalculator.class, ResourceCalculator.class);
- newConf
- .setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- // Set maximum capacity of queue "a" to 50
- newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 50);
- MockRM rm1 = new MockRM(newConf);
-
- RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
- nodeLabelsManager.init(newConf);
- rm1.getRMContext().setNodeLabelManager(nodeLabelsManager);
- rm1.start();
- MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
- MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
-
- // launch an app to queue "a", AM container should be launched on nm1
- RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
- MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
-
- // launch another app to queue "b", AM container should be launched on nm1
- RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b");
- MockRM.launchAndRegisterAM(app2, rm1, nm1);
-
- am1.allocate("*", 7 * GB, 2, new ArrayList<ContainerId>());
-
- CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
- RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
- RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
- FiCaSchedulerApp schedulerApp1 =
- cs.getApplicationAttempt(am1.getApplicationAttemptId());
-
- // Do nm1 heartbeats 1 times, will reserve a container on nm1 for app1
- cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
- Assert.assertEquals(1, schedulerApp1.getReservedContainers().size());
-
- // Do nm2 heartbeats 1 times, will unreserve a container on nm1
- // and allocate a container on nm2 for app1
- cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
- Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
- Assert.assertEquals(0, schedulerApp1.getReservedContainers().size());
-
- rm1.close();
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org