You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/28 22:27:28 UTC
[01/24] hadoop git commit: YARN-1015. FS should watch node resource
utilization and allocate opportunistic containers if appropriate. [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 954986dd5 -> 8d217ee3c (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 4b637ff..ff620b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -57,8 +57,13 @@ import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -66,6 +71,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -77,6 +83,8 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -99,6 +107,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -1062,15 +1071,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(2, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// verify metrics
QueueMetrics queue1Metrics = scheduler.getQueueManager().getQueue("queue1")
@@ -1105,7 +1114,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1115,7 +1124,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
// Now another node checks in with capacity
@@ -1129,7 +1138,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// The old reservation should still be there...
assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemorySize());
@@ -1139,7 +1148,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
- @Test (timeout = 5000)
+ @Test
public void testOffSwitchAppReservationThreshold() throws Exception {
conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f);
scheduler.init(conf);
@@ -1179,7 +1188,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(6144, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1211,7 +1220,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1272,7 +1281,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Create new app with a resource request that can be satisfied by any
// node but would be
@@ -1317,7 +1326,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(new NodeUpdateSchedulerEvent(node4));
assertEquals(10240, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
scheduler.handle(new NodeUpdateSchedulerEvent(node1));
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -1361,7 +1370,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Verify capacity allocation
assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Verify number of reservations have decremented
assertEquals(0,
@@ -1405,7 +1414,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1414,7 +1423,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1540,7 +1549,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests likewise
createSchedulingRequest(1024, "queue2", "user2", 1);
@@ -1549,7 +1558,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is allocated app capacity
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
scheduler.update();
@@ -1589,12 +1598,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure allocated memory of queue1 doesn't exceed its maximum
assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//the reservation of queue1 should be reclaim
assertEquals(0, scheduler.getSchedulerApp(attId1).
getCurrentReservation().getMemorySize());
assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
}
@Test
@@ -1634,7 +1643,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 1 is allocated app capacity
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// Now queue 2 requests below threshold
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1);
@@ -1643,7 +1652,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 has no reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(0,
scheduler.getSchedulerApp(attId).getReservedContainers().size());
@@ -1654,7 +1663,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
.getVirtualCores());
@@ -1669,7 +1678,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Make sure this goes to queue 2
assertEquals(3, scheduler.getQueueManager().getQueue("queue2").
- getResourceUsage().getVirtualCores());
+ getGuaranteedResourceUsage().getVirtualCores());
// The old reservation should still be there...
assertEquals(3, scheduler.getSchedulerApp(attId).getCurrentReservation()
@@ -2702,7 +2711,361 @@ public class TestFairScheduler extends FairSchedulerTestBase {
2, liveContainers.iterator().next().getContainer().
getPriority().getPriority());
}
-
+
+ /**
+ * Test that NO OPPORTUNISTIC containers can be allocated on a node that
+ * is fully allocated and with a very high utilization.
+ */
+ @Test
+ public void testAllocateNoOpportunisticContainersOnBusyNode()
+ throws IOException {
+ conf.setBoolean(
+ YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED, true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(2048, 2), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the node's full memory
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization shoots up after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(2000, 0, 0.8f));
+
+ // create another scheduling request
+ ApplicationAttemptId appAttempt2
+ = createSchedulingRequest(100, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue("Expecting no containers allocated",
+ allocatedContainers2.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that a reservation is made for the second resource request
+ Resource reserved = scheduler.getNode(node.getNodeID()).
+ getReservedContainer().getReservedResource();
+ assertTrue("Expect a reservation made for the second resource request",
+ reserved.equals(Resource.newInstance(100, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test that OPPORTUNISTIC containers can be allocated on a node with low
+ * utilization even though there is not enough unallocated resource on the
+ * node to accommodate the request.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnPartiallyOverAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node that is fully
+ * allocated but whose utilization is very low.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersOnFullyAllocatedNode()
+ throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 = createSchedulingRequest(
+ 4096, "queue1", "user1", 4);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1800, 0, 0.5f));
+
+ // create another scheduling request now that there is no unallocated
+ // resources left on the node, the request should be served with an
+ // allocation of an opportunistic container
+ ApplicationAttemptId appAttempt2 = createSchedulingRequest(
+ 1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // verify that no reservation is made for the second request given
+ // that it's satisfied by an OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made because we have satisfied" +
+ " the second request with an OPPORTUNISTIC container allocation",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test opportunistic containers can be allocated on a node with a low
+ * utilization even though there are GUARANTEED containers allocated.
+ */
+ @Test
+ public void testAllocateOpportunisticContainersWithGuaranteedOnes()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3200, "queue1", "user1", 3);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3200, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container is launched on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(512, 0, 0.1f));
+
+ // create two other scheduling requests which in aggregate ask for more
+ // that what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(512, "queue2", "user1", 1);
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(512, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // verify that no reservation is made given that the second request should
+ // be satisfied by a GUARANTEED container allocation, the third by an
+ // OPPORTUNISTIC container allocation.
+ assertTrue("No reservation should be made.",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
@@ -3692,7 +4055,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.createAbnormalContainerStatus(container.getContainerId(),
SchedulerUtils.COMPLETED_APPLICATION),
RMContainerEventType.FINISHED);
- assertEquals(Resources.none(), app1.getResourceUsage());
+ assertEquals(Resources.none(), app1.getGuaranteedResourceUsage());
}
@Test
@@ -3792,7 +4155,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application1's AM should be finished",
0, app1.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app1.getResourceUsage());
+ Resources.none(), app1.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be running",
1, app3.getLiveContainers().size());
assertEquals("Application3's AM requests 1024 MB memory",
@@ -3812,7 +4175,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application4's AM should not be running",
0, app4.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app4.getResourceUsage());
+ Resources.none(), app4.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3828,7 +4191,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3841,7 +4204,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should not be running",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
@@ -3857,11 +4220,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application2's AM should be finished",
0, app2.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app2.getResourceUsage());
+ Resources.none(), app2.getGuaranteedResourceUsage());
assertEquals("Application3's AM should be finished",
0, app3.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app3.getResourceUsage());
+ Resources.none(), app3.getGuaranteedResourceUsage());
assertEquals("Application5's AM should be running",
1, app5.getLiveContainers().size());
assertEquals("Application5's AM requests 2048 MB memory",
@@ -3882,7 +4245,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application5's AM should have 0 container",
0, app5.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app5.getResourceUsage());
+ Resources.none(), app5.getGuaranteedResourceUsage());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
2048, queue1.getAmResourceUsage().getMemorySize());
scheduler.update();
@@ -3906,7 +4269,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("Application6's AM should not be running",
0, app6.getLiveContainers().size());
assertEquals("Finished application usage should be none",
- Resources.none(), app6.getResourceUsage());
+ Resources.none(), app6.getGuaranteedResourceUsage());
assertEquals("Application6's AM resource shouldn't be updated",
0, app6.getAMResource().getMemorySize());
assertEquals("Queue1's AM resource usage should be 2048 MB memory",
@@ -4621,17 +4984,25 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true);
FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true);
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2");
- Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB);
- Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0);
- Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(parent2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(queue2.getGuaranteedResourceUsage().getMemorySize(),
+ 1 * GB);
+ Assert.assertEquals(parent1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
+ Assert.assertEquals(queue1.getGuaranteedResourceUsage().getMemorySize(),
+ 0);
}
@Test (expected = YarnException.class)
@@ -4671,7 +5042,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(2048, 2),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
@@ -5117,7 +5489,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
//container will be reserved at node1
RMContainer reservedContainer1 =
@@ -5137,7 +5509,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
app1, RMAppAttemptState.KILLED, false));
assertEquals(0, scheduler.getQueueManager().getQueue("queue1").
- getResourceUsage().getMemorySize());
+ getGuaranteedResourceUsage().getMemorySize());
// container will be allocated at node2
scheduler.handle(new NodeUpdateSchedulerEvent(node2));
@@ -5285,10 +5657,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app1 = mock(FSAppAttempt.class);
Mockito.when(app1.getDemand()).thenReturn(maxResource);
- Mockito.when(app1.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app1.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
FSAppAttempt app2 = mock(FSAppAttempt.class);
Mockito.when(app2.getDemand()).thenReturn(maxResource);
- Mockito.when(app2.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app2.getGuaranteedResourceUsage()).
+ thenReturn(Resources.none());
QueueManager queueManager = scheduler.getQueueManager();
FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
@@ -5344,7 +5718,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
child1.setMaxShare(new ConfigurableResource(resource));
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(resource);
- Mockito.when(app.getResourceUsage()).thenReturn(resource);
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(resource);
child1.addApp(app, true);
child1.updateDemand();
@@ -5380,7 +5754,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
+ " SteadyFairShare: <memory:0, vCores:0>,"
+ " MaxShare: <memory:4096, vCores:4>,"
+ " MinShare: <memory:0, vCores:0>,"
- + " ResourceUsage: <memory:4096, vCores:4>,"
+ + " Guaranteed ResourceUsage: <memory:4096, vCores:4>,"
+ " Demand: <memory:4096, vCores:4>,"
+ " MaxAMShare: 0.5,"
+ " Runnable: 0}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index b016c1b..6777b5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -243,11 +243,16 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public Resource getMinShare() {
return minShare;
}
@@ -278,7 +283,8 @@ public class TestSchedulingPolicy {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node,
+ boolean opportunistic) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
index de4d5a1..75581cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
@@ -253,7 +253,7 @@ public class TestRMWebServicesFairSchedulerCustomResourceTypes
final long value) {
try {
Method incUsedResourceMethod = queue.getClass().getSuperclass()
- .getDeclaredMethod("incUsedResource", Resource.class);
+ .getDeclaredMethod("incUsedGuaranteedResource", Resource.class);
incUsedResourceMethod.setAccessible(true);
Map<String, Long> customResources =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/24] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c045be51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c045be51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c045be51
Branch: refs/heads/YARN-1011
Commit: c045be51553f0faa61c5bd0c7b52d2d785fa57d2
Parents: 8757aa9
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Jan 23 22:34:49 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:14:49 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/TypeConverter.java | 4 +-
.../hadoop/mapreduce/TestTypeConverter.java | 4 +-
.../hadoop/mapred/TestResourceMgrDelegate.java | 2 +-
.../records/ApplicationResourceUsageReport.java | 158 +++++++++++---
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 23 ++-
.../ApplicationResourceUsageReportPBImpl.java | 205 +++++++++++++++----
.../hadoop/yarn/util/resource/Resources.java | 20 ++
...pplicationHistoryManagerOnTimelineStore.java | 83 +++++---
.../TestApplicationHistoryClientService.java | 8 +-
...pplicationHistoryManagerOnTimelineStore.java | 18 +-
.../metrics/ApplicationMetricsConstants.java | 10 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 39 +++-
.../server/resourcemanager/RMAppManager.java | 18 +-
.../server/resourcemanager/RMServerUtils.java | 3 +-
.../metrics/TimelineServiceV1Publisher.java | 14 +-
.../metrics/TimelineServiceV2Publisher.java | 14 +-
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../records/ApplicationAttemptStateData.java | 144 +++++++++++--
.../pb/ApplicationAttemptStateDataPBImpl.java | 110 ++++++++--
.../server/resourcemanager/rmapp/RMAppImpl.java | 38 ++--
.../resourcemanager/rmapp/RMAppMetrics.java | 38 ++--
.../attempt/AggregateAppResourceUsage.java | 51 +++--
.../rmapp/attempt/RMAppAttemptImpl.java | 27 ++-
.../rmapp/attempt/RMAppAttemptMetrics.java | 49 +++--
.../ContainerResourceUsageReport.java | 46 +++++
.../rmcontainer/RMContainer.java | 4 +-
.../rmcontainer/RMContainerImpl.java | 117 ++++++++---
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 62 ++++--
.../scheduler/YarnScheduler.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 9 +-
.../webapp/FairSchedulerAppsBlock.java | 8 +
.../resourcemanager/webapp/RMAppBlock.java | 9 +-
.../resourcemanager/webapp/RMAppsBlock.java | 10 +
.../resourcemanager/webapp/dao/AppInfo.java | 78 +++++--
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/TestAppManager.java | 39 ++--
.../resourcemanager/TestApplicationACLs.java | 4 +-
.../resourcemanager/TestClientRMService.java | 45 ++--
.../TestContainerResourceUsage.java | 184 ++++++++++-------
.../applicationsmanager/MockAsm.java | 4 +-
.../TestCombinedSystemMetricsPublisher.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 23 ++-
.../TestSystemMetricsPublisherForV2.java | 12 +-
.../recovery/RMStateStoreTestBase.java | 12 +-
.../recovery/TestZKRMStateStore.java | 40 ++--
.../rmapp/TestRMAppTransitions.java | 6 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +--
.../capacity/TestCapacityScheduler.java | 4 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 20 +-
.../resourcemanager/webapp/TestAppPage.java | 2 +-
.../resourcemanager/webapp/TestRMWebApp.java | 3 +-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
...estRMWebServicesAppsCustomResourceTypes.java | 2 +-
.../router/webapp/RouterWebServiceUtil.java | 14 +-
.../router/webapp/TestRouterWebServiceUtil.java | 14 +-
62 files changed, 1432 insertions(+), 515 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index aea931a..23409fc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -527,8 +527,8 @@ public class TypeConverter {
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
(int)resourceUsageReport.getReservedResources().getMemorySize());
- jobStatus.setUsedMem(
- (int) resourceUsageReport.getUsedResources().getMemorySize());
+ jobStatus.setUsedMem((int) resourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
}
return jobStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
index 629a246..acd9c35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
@@ -94,7 +94,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
applicationReport.setApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
Assert.assertEquals(appStartTime, jobStatus.getStartTime());
@@ -133,7 +133,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt);
JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
Assert.assertNotNull("fromYarn returned null status", status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
index d6754f0..0506b27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
@@ -137,7 +137,7 @@ public class TestResourceMgrDelegate {
Records.newRecord(Resource.class));
Mockito.when(appResources.getReservedResources()).thenReturn(
Records.newRecord(Resource.class));
- Mockito.when(appResources.getUsedResources()).thenReturn(
+ Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn(
Records.newRecord(Resource.class));
Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
appResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index d2e33ff..4517db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,21 +36,27 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public static ApplicationResourceUsageReport newInstance(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap, float queueUsagePerc,
- float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap, float queueUsagePerc,
+ float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
+
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
@@ -87,16 +93,40 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNumReservedContainers(int num_reserved_containers);
/**
- * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports.
- * @return the used <code>Resource</code>
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsedResources();
+ /**
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourcesUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourcesUsed(Resource resources);
+
+ /**
+ * Get the opportunistic <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the opportunistic <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourcesUsed();
+
@Private
@Unstable
- public abstract void setUsedResources(Resource resources);
+ public abstract void setOpportunisticResourcesUsed(Resource resources);
/**
* Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports.
@@ -123,42 +153,84 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNeededResources(Resource needed_resources);
/**
- * Set the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @param memory_seconds the aggregated amount of memory seconds
+ * Set the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @param memorySeconds the aggregated amount of guaranteed memory seconds
*/
@Private
@Unstable
- public abstract void setMemorySeconds(long memory_seconds);
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
/**
- * Get the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @return the aggregated amount of memory seconds
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ /**
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
/**
- * Set the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @param vcore_seconds the aggregated number of vcore seconds
+ * Set the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @param vcoreSeconds the aggregated number of guaranteed vcore seconds
*/
@Private
@Unstable
- public abstract void setVcoreSeconds(long vcore_seconds);
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @return the aggregated number of vcore seconds
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ /**
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
/**
+ * Get the aggregated amount of opportunistic memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of opportunistic memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the aggregated number of opportunistic vcores that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return the aggregated number of opportunistic vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the percentage of resources of the queue that the app is using.
* @return the percentage of resources of the queue that the app is using.
*/
@@ -231,23 +303,35 @@ public abstract class ApplicationResourceUsageReport {
public abstract long getPreemptedVcoreSeconds();
/**
- * Get the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
- * @return map containing the resource name and aggregated resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Private
@Unstable
- public abstract void setResourceSecondsMap(
+ public abstract void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
@@ -272,4 +356,24 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @param opportunisticResourceSecondsMap map containing the resource name
+ * and aggregated opportunistic resource-seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 49bf895..3c972b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -252,6 +252,8 @@ message ApplicationResourceUsageReportProto {
optional int64 preempted_vcore_seconds = 11;
repeated StringLongMapProto application_resource_usage_map = 12;
repeated StringLongMapProto application_preempted_resource_usage_map = 13;
+ optional ResourceProto used_opportunistic_resources = 14;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 15;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index a0e4e02..063c282 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -1144,15 +1144,20 @@ public class ApplicationCLI extends YarnCLI {
private void printResourceUsage(PrintWriter appReportStr,
ApplicationResourceUsageReport usageReport) {
- appReportStr.print("\tAggregate Resource Allocation : ");
+ appReportStr.print("\tAggregate Guaranteed Resource Allocation : ");
if (usageReport != null) {
- appReportStr.println(
- getResourceSecondsString(usageReport.getResourceSecondsMap()));
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getGuaranteedResourceSecondsMap()));
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getOpportunisticResourceSecondsMap()));
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println(getResourceSecondsString(
usageReport.getPreemptedResourceSecondsMap()));
} else {
appReportStr.println("N/A");
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index aed5258..27f276b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -170,13 +170,11 @@ public class TopCLI extends YarnCLI {
appReport.getApplicationResourceUsageReport()
.getNumReservedContainers();
displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers));
- usedVirtualCores =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getVirtualCores();
+ usedVirtualCores = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getVirtualCores();
displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores));
- usedMemory =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getMemorySize() / 1024;
+ usedMemory = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getMemorySize() / 1024;
displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G");
reservedVirtualCores =
appReport.getApplicationResourceUsageReport().getReservedResources()
@@ -195,11 +193,11 @@ public class TopCLI extends YarnCLI {
progress = appReport.getProgress() * 100;
displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress));
// store in GBSeconds
- memorySeconds =
- appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024;
+ memorySeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedMemorySeconds() / 1024;
displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds));
- vcoreSeconds =
- appReport.getApplicationResourceUsageReport().getVcoreSeconds();
+ vcoreSeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedVcoreSeconds();
displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 6a24837..4778803 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -123,18 +123,27 @@ public class TestYarnCLI {
for (int i = 0; i < 2; ++i) {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
preemptedResoureSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 2222L);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport
- .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(2, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
@@ -170,7 +179,9 @@ public class TestYarnCLI {
pw.println("\tTracking-URL : N/A");
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
- pw.println("\tAggregate Resource Allocation : " +
+ pw.println("\tAggregate Guaranteed Resource Allocation : " +
+ (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tAggregate Opportunistic Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tAggregate Resource Preempted : " +
(i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 14ede5d..8ff1631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -40,12 +40,14 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto.Builder builder = null;
boolean viaProto = false;
- Resource usedResources;
- Resource reservedResources;
- Resource neededResources;
+ private Resource guaranteedResourceUsed;
+ private Resource reservedResources;
+ private Resource neededResources;
+ private Resource opportunisticResourceUsed;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationResourceUsageReportPBImpl() {
builder = ApplicationResourceUsageReportProto.newBuilder();
@@ -55,8 +57,9 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto proto) {
this.proto = proto;
viaProto = true;
- getResourceSecondsMap();
+ getGuaranteedResourceSecondsMap();
getPreemptedResourceSecondsMap();
+ getOpportunisticResourceSecondsMap();
}
public synchronized ApplicationResourceUsageReportProto getProto() {
@@ -87,8 +90,13 @@ extends ApplicationResourceUsageReport {
}
private void mergeLocalToBuilder() {
- if (this.usedResources != null) {
- builder.setUsedResources(convertToProtoFormat(this.usedResources));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setUsedResources(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setUsedOpportunisticResources(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.reservedResources != null) {
builder.setReservedResources(
@@ -99,19 +107,28 @@ extends ApplicationResourceUsageReport {
}
builder.clearApplicationResourceUsageMap();
builder.clearApplicationPreemptedResourceUsageMap();
+ builder.clearApplicationOpportunisticResourceUsageMap();
if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap
.isEmpty()) {
builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
- if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) {
+ if (guaranteedResourceSecondsMap != null &&
+ !guaranteedResourceSecondsMap.isEmpty()) {
builder.addAllApplicationResourceUsageMap(
- ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ guaranteedResourceSecondsMap));
+ }
+ if (opportunisticResourceSecondsMap != null &&
+ !opportunisticResourceSecondsMap.isEmpty()) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ opportunisticResourceSecondsMap));
}
- builder.setMemorySeconds(this.getMemorySeconds());
- builder.setVcoreSeconds(this.getVcoreSeconds());
+ builder.setMemorySeconds(this.getGuaranteedMemorySeconds());
+ builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds());
builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds());
builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds());
}
@@ -157,24 +174,53 @@ extends ApplicationResourceUsageReport {
}
@Override
+ @Deprecated
public synchronized Resource getUsedResources() {
+ return getGuaranteedResourcesUsed();
+ }
+
+ @Override
+ public synchronized Resource getGuaranteedResourcesUsed() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- if (this.usedResources != null) {
- return this.usedResources;
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
}
if (!p.hasUsedResources()) {
return null;
}
- this.usedResources = convertFromProtoFormat(p.getUsedResources());
- return this.usedResources;
+ this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources());
+ return this.guaranteedResourceUsed;
}
@Override
- public synchronized void setUsedResources(Resource resources) {
+ public synchronized void setGuaranteedResourcesUsed(Resource resources) {
maybeInitBuilder();
if (resources == null)
builder.clearUsedResources();
- this.usedResources = resources;
+ this.guaranteedResourceUsed = resources;
+ }
+
+ @Override
+ public synchronized Resource getOpportunisticResourcesUsed() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
+ }
+ if (!p.hasUsedOpportunisticResources()) {
+ return null;
+ }
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getUsedOpportunisticResources());
+ return this.opportunisticResourceUsed;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourcesUsed(Resource resources) {
+ maybeInitBuilder();
+ if (resources == null) {
+ builder.clearUsedOpportunisticResources();
+ }
+ this.opportunisticResourceUsed = resources;
}
@Override
@@ -220,14 +266,14 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setMemorySeconds(long memory_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds);
+ public synchronized void setGuaranteedMemorySeconds(long memorySeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
}
@Override
- public synchronized long getMemorySeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ public synchronized long getGuaranteedMemorySeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
@@ -235,20 +281,50 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setVcoreSeconds(long vcore_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.VCORES.getName(), vcore_seconds);
+ @Deprecated
+ public synchronized long getMemorySeconds() {
+ return getGuaranteedMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
}
@Override
+ public synchronized long getGuaranteedVcoreSeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ @Deprecated
public synchronized long getVcoreSeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ return getGuaranteedVcoreSeconds();
+ }
+
+ @Override
+ public synchronized long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public synchronized long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
-
+
@Override
public synchronized void setPreemptedMemorySeconds(
long preemptedMemorySeconds) {
@@ -314,41 +390,47 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setResourceSecondsMap(
+ public synchronized void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap == null) {
return;
}
if (!resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(0L);
+ this.setGuaranteedMemorySeconds(0L);
}
if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(0L);
+ this.setGuaranteedVcoreSeconds(0L);
}
}
@Override
+ @Deprecated
public synchronized Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ public synchronized Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils
+ this.guaranteedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
}
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(p.getVcoreSeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
}
- this.setMemorySeconds(p.getMemorySeconds());
- this.setVcoreSeconds(p.getVcoreSeconds());
- return this.resourceSecondsMap;
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
+ return this.guaranteedResourceSecondsMap;
}
@Override
@@ -389,4 +471,45 @@ extends ApplicationResourceUsageReport {
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
return this.preemptedResourceSecondsMap;
}
+
+ @Override
+ public synchronized Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap = ProtoUtils
+ .convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ if (opportunisticResourceSecondsMap == null) {
+ return;
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 48c2c36..6c41015 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import java.util.Map;
+
/**
* Resources is a computation class which provides a set of apis to do
* mathematical operations on Resource object.
@@ -553,4 +555,22 @@ public class Resources {
Resource resource, Resource factor) {
return calculator.normalizeDown(resource, factor);
}
+
+ /**
+ * Merge resource usage entries from a map to another map.
+ * @param mergeFrom the map to merge from
+ * @param mergeTo the map to merge to
+ */
+ public static void mergeResourceSecondsMap(Map<String, Long> mergeFrom,
+ Map<String, Long> mergeTo) {
+ for (Map.Entry<String, Long> entry : mergeFrom.entrySet()) {
+ Long value = mergeTo.get(entry.getKey());
+ if (value != null) {
+ value += entry.getValue();
+ } else {
+ value = entry.getValue();
+ }
+ mergeTo.put(entry.getKey(), value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 0b57717..85cc006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -330,29 +329,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
.toString();
}
- if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_METRICS);
- long memorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_METRICS);
- long preemptedMemorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
- long preemptedVcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- preemptedMemorySeconds);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
-
+ if (entityInfo.containsKey(
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) {
+ Map<String, Long> guaranteedResourceSecondsMap =
+ extractGuaranteedResourceSecondsMap(entityInfo);
+ Map<String, Long> preemptedResoureSecondsMap =
+ extractPreemptedResourceSecondsMap(entityInfo);
+ Map<String, Long> opportunisticResourceSecondsMap =
+ extractOpportunisticResourceSecondsMap(entityInfo);
appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(0, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
@@ -456,6 +445,54 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static Map<String, Long> extractGuaranteedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+
+ long guaranteedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS);
+ long guaranteedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds);
+ guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(),
+ guaranteedVcoreSeconds);
+
+ return guaranteedResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractOpportunisticResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+
+ long opportunisticVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS);
+ long opportunisticMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(),
+ opportunisticMemorySeconds);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds);
+
+ return opportunisticResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractPreemptedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
+
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
+ preemptedMemorySeconds);
+ preemptedResoureSecondsMap
+ .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
+
+ return preemptedResoureSecondsMap;
+ }
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7ef6eca..6cb21ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -152,9 +152,13 @@ public class TestApplicationHistoryClientService {
ApplicationReport appReport = response.getApplicationReport();
Assert.assertNotNull(appReport);
Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
- .getMemorySeconds());
+ .getGuaranteedMemorySeconds());
Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
- .getVcoreSeconds());
+ .getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticMemorySeconds());
+ Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticVcoreSeconds());
Assert.assertEquals("application_0_0001", appReport.getApplicationId()
.toString());
Assert.assertEquals("test app type",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ecaaf1e..35996aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -250,9 +250,13 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
- applicationResourceUsageReport.getMemorySeconds());
- Assert
- .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+ applicationResourceUsageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123,
+ applicationResourceUsageReport.getOpportunisticMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getOpportunisticVcoreSeconds());
long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
@@ -534,8 +538,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345);
if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 4cec409..b4123ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -70,12 +70,18 @@ public class ApplicationMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_APPLICATION_STATE";
- public static final String APP_CPU_METRICS =
+ public static final String APP_GUARANTEED_CPU_METRICS =
"YARN_APPLICATION_CPU";
- public static final String APP_MEM_METRICS =
+ public static final String APP_GUARANTEED_MEM_METRICS =
"YARN_APPLICATION_MEMORY";
+ public static final String APP_OPPORTUNISTIC_CPU_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_CPU";
+
+ public static final String APP_OPPORTUNISTIC_MEM_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_MEMORY";
+
public static final String APP_RESOURCE_PREEMPTED_CPU =
"YARN_APPLICATION_RESOURCE_PREEMPTED_CPU";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index a507e8d..c339b80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -456,19 +456,24 @@ public class BuilderUtils {
}
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index b4687a4..be6c17c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Times;
@@ -62,6 +63,8 @@ public class AppInfo {
protected int priority;
private long allocatedCpuVcores;
private long allocatedMemoryMB;
+ private long allocatedOpportunisticCpuVcores;
+ private long allocatedOpportunisticMemoryMB;
private long reservedCpuVcores;
private long reservedMemoryMB;
protected boolean unmanagedApplication;
@@ -100,15 +103,25 @@ public class AppInfo {
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
- if (app.getApplicationResourceUsageReport().getUsedResources() != null) {
- allocatedCpuVcores = app.getApplicationResourceUsageReport()
- .getUsedResources().getVirtualCores();
- allocatedMemoryMB = app.getApplicationResourceUsageReport()
- .getUsedResources().getMemorySize();
- reservedCpuVcores = app.getApplicationResourceUsageReport()
- .getReservedResources().getVirtualCores();
- reservedMemoryMB = app.getApplicationResourceUsageReport()
- .getReservedResources().getMemorySize();
+ Resource guaranteedResourceUsed =
+ app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed();
+ if (guaranteedResourceUsed != null) {
+ allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores();
+ allocatedMemoryMB = guaranteedResourceUsed.getMemorySize();
+ }
+ Resource opportunisticResourceUsed = app
+ .getApplicationResourceUsageReport().getOpportunisticResourcesUsed();
+ if (opportunisticResourceUsed != null) {
+ allocatedOpportunisticCpuVcores =
+ opportunisticResourceUsed.getVirtualCores();
+ allocatedOpportunisticMemoryMB =
+ opportunisticResourceUsed.getMemorySize();
+ }
+ Resource resourceReserved =
+ app.getApplicationResourceUsageReport().getReservedResources();
+ if (resourceReserved != null) {
+ reservedCpuVcores = resourceReserved.getVirtualCores();
+ reservedMemoryMB = resourceReserved.getMemorySize();
}
}
progress = app.getProgress() * 100; // in percent
@@ -168,6 +181,14 @@ public class AppInfo {
return allocatedMemoryMB;
}
+ public long getAllocatedOpportunisticCpuVcores() {
+ return allocatedOpportunisticCpuVcores;
+ }
+
+ public long getAllocatedOpportunisticMemoryMB() {
+ return allocatedOpportunisticMemoryMB;
+ }
+
public long getReservedCpuVcores() {
return reservedCpuVcores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 6ea55ed..b36d220 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -186,19 +186,27 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
- .add("memorySeconds", metrics.getMemorySeconds())
- .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds())
+ .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds())
+ .add("opportunisticMemorySeconds",
+ metrics.getOpportunisticMemorySeconds())
+ .add("opportunisticVcoreSeconds",
+ metrics.getOpportunisticVcoreSeconds())
.add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds())
.add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds())
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
.add("applicationType", app.getApplicationType())
- .add("resourceSeconds", StringHelper
- .getResourceSecondsString(metrics.getResourceSecondsMap()))
+ .add("guaranteedResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getGuaranteedResourceSecondsMap()))
.add("preemptedResourceSeconds", StringHelper
.getResourceSecondsString(
- metrics.getPreemptedResourceSecondsMap()));
+ metrics.getPreemptedResourceSecondsMap()))
+ .add("opportunisticResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getOpportunisticResourceSecondsMap()));
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index ab6bbcf..bf93df2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -481,7 +481,8 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>());
+ Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(),
+ Resources.createResource(-1, -1), new HashMap<>());
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index dc5292b..909f7a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -142,10 +142,16 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
latestApplicationAttemptId);
}
RMAppMetrics appMetrics = app.getRMAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS,
+ appMetrics.getGuaranteedVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS,
+ appMetrics.getGuaranteedMemorySeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ appMetrics.getOpportunisticVcoreSeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ appMetrics.getOpportunisticMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,
appMetrics.getPreemptedMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 89905e5..7a21e55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -193,11 +193,17 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_CPU_METRICS, timestamp,
- appMetrics.getVcoreSeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp,
+ appMetrics.getGuaranteedVcoreSeconds()));
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_MEM_METRICS, timestamp,
- appMetrics.getMemorySeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp,
+ appMetrics.getGuaranteedMemorySeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ timestamp, appMetrics.getOpportunisticVcoreSeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ timestamp, appMetrics.getOpportunisticMemorySeconds()));
entityMetrics.add(getTimelineMetric(
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp,
appMetrics.getPreemptedMemorySeconds()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 242b5d0..4a7696f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -904,8 +904,9 @@ public abstract class RMStateStore extends AbstractService {
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
- resUsage.getResourceUsageSecondsMap(),
- attempMetrics.getPreemptedResourceSecondsMap());
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ attempMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
getRMStateStoreEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 2de071a..b08d2eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -45,8 +46,9 @@ public abstract class ApplicationAttemptStateData {
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
- long finishTime, Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ long finishTime, Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -59,19 +61,22 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
attemptStateData.setFinishTime(finishTime);
- attemptStateData.setMemorySeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap,
+ attemptStateData.setGuaranteedMemorySeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
- attemptStateData.setVcoreSeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L));
+ attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L));
attemptStateData.setPreemptedMemorySeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setPreemptedVcoreSeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L));
- attemptStateData.setResourceSecondsMap(resourceSecondsMap);
+ attemptStateData.setGuaranteedResourceSecondsMap(
+ guaranteedResourceSecondsMap);
+ attemptStateData.setOpportunisticResourceSecondsMap(
+ opportunisticResourcesSecondsMap);
attemptStateData
.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return attemptStateData;
@@ -80,11 +85,13 @@ public abstract class ApplicationAttemptStateData {
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime,
- Map<String, Long> resourceSeondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
return newInstance(attemptId, masterContainer, attemptTokens, startTime,
null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
- resourceSeondsMap, preemptedResourceSecondsMap);
+ guaranteedResourceSecondsMap, preemptedResourceSecondsMap,
+ opportunisticResourcesSecondsMap);
}
@@ -180,30 +187,78 @@ public abstract class ApplicationAttemptStateData {
public abstract void setFinishTime(long finishTime);
/**
- * Get the <em>memory seconds</em> (in MB seconds) of the application.
- * @return <em>memory seconds</em> (in MB seconds) of the application
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setMemorySeconds(long memorySeconds);
/**
- * Get the <em>vcore seconds</em> of the application.
- * @return <em>vcore seconds</em> of the application
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setVcoreSeconds(long vcoreSeconds);
/**
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ @Private
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ @Private
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the <em>opportunistic vcore seconds</em> of the application.
+ * @return <em>opportunistic vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the <em>preempted memory seconds</em>
* (in MB seconds) of the application.
* @return <em>preempted memory seconds</em>
@@ -232,28 +287,54 @@ public abstract class ApplicationAttemptStateData {
public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of resources preempted that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
- * @return map containing the resource name and aggregated preempted
- * resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
+
+ /**
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * guaranteed resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
+
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
@@ -276,4 +357,27 @@ public abstract class ApplicationAttemptStateData {
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * opportunistic resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/24] hadoop git commit: Revert "Revert "YARN-6675. Add NM support
to launch opportunistic containers based on overallocation. Contributed by
Haibo Chen.""
Posted by ha...@apache.org.
Revert "Revert "YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen.""
This reverts commit 1235cf5deb9f309e975c4f59ee27caf1282f19f9.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9db0368
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9db0368
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9db0368
Branch: refs/heads/YARN-1011
Commit: f9db0368af37e6257b99cc794a0cac43089c88d2
Parents: ddae86c
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jun 11 13:34:40 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:00 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 ++
...locationBasedResourceUtilizationTracker.java | 158 ---
.../scheduler/ContainerScheduler.java | 317 +++--
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 +
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 +
.../UtilizationBasedResourceTracker.java | 95 ++
.../BaseContainerManagerTest.java | 35 +
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 ++
...locationBasedResourceUtilizationTracker.java | 93 --
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ++++++++++++++++++
20 files changed, 1916 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 572684e..d757376 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization();
+ return containersMonitor.getContainersUtilization(false).getUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 27a7c80..a08e227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,6 +245,12 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
+
+ // ContainersLauncher must be added after ContainersMonitor
+ // because the former depends on the latter to initialize
+ // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -269,8 +275,6 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..0228332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ protected String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..2f5acfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,8 +121,7 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
- event.getContainer(), dirsHandler, containerManager);
+ createContainerLaunch(app, event.getContainer());
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -225,4 +224,10 @@ public class ContainersLauncher extends AbstractService
break;
}
}
+
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 64831e9..8da4ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,10 +23,24 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
- ResourceUtilization getContainersUtilization();
+
+ /**
+ * Get the aggregate resource utilization of containers running on the node,
+ * with a timestamp of the measurement.
+ * @param latest true if the latest result should be returned
+ * @return ResourceUtilization resource utilization of all containers
+ */
+ ContainersResourceUtilization getContainersUtilization(boolean latest);
+
+ /**
+ * Get the policy to over-allocate containers when over-allocation is on.
+ * @return null if over-allocation is turned off
+ */
+ NMAllocationPolicy getContainerOverAllocationPolicy();
float getVmemRatio();
@@ -66,4 +80,26 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
+
+ /**
+ * A snapshot of resource utilization of all containers with the timestamp.
+ */
+ final class ContainersResourceUtilization {
+ private final ResourceUtilization utilization;
+ private final long timestamp;
+
+ public ContainersResourceUtilization(
+ ResourceUtilization utilization, long timestamp) {
+ this.utilization = utilization;
+ this.timestamp = timestamp;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public ResourceUtilization getUtilization() {
+ return utilization;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 7873882..a045d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,8 +116,9 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ResourceUtilization containersUtilization;
+ private ContainersResourceUtilization latestContainersUtilization;
+ private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -129,7 +134,8 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
}
@Override
@@ -365,6 +371,10 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
+ // TODO make this configurable
+ this.overAllocationPolicy =
+ createOverAllocationPolicy(resourceThresholds);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -372,6 +382,11 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
+ protected NMAllocationPolicy createOverAllocationPolicy(
+ ResourceThresholds resourceThresholds) {
+ return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -655,7 +670,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setContainersUtilization(trackedContainersUtilization);
+ setLatestContainersUtilization(trackedContainersUtilization);
+
+ // check opportunity to start containers if over-allocation is on
+ if (context.isOverAllocationEnabled()) {
+ attemptToStartContainersUponLowUtilization();
+ }
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1045,12 +1065,34 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ResourceUtilization getContainersUtilization() {
- return this.containersUtilization;
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ // TODO update containerUtilization if latest is true
+ return this.latestContainersUtilization;
+ }
+
+ @Override
+ public NMAllocationPolicy getContainerOverAllocationPolicy() {
+ return overAllocationPolicy;
+ }
+
+ private void setLatestContainersUtilization(ResourceUtilization utilization) {
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ utilization, System.currentTimeMillis());
}
- private void setContainersUtilization(ResourceUtilization utilization) {
- this.containersUtilization = utilization;
+ @VisibleForTesting
+ public void attemptToStartContainersUponLowUtilization() {
+ if (getContainerOverAllocationPolicy() != null) {
+ Resource available = getContainerOverAllocationPolicy()
+ .getAvailableResources();
+ if (available.getMemorySize() > 0 &&
+ available.getVirtualCores() > 0) {
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(null,
+ ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
new file mode 100644
index 0000000..86b3698
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the resource utilization tracker that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceTracker
+ implements ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private static final Resource UNAVAILABLE =
+ Resource.newInstance(0, 0);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+
+ AllocationBasedResourceTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to containers.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Get the amount of resources that have not been allocated to containers.
+ * @return Resource resources that have not been allocated to containers.
+ */
+ protected Resource getUnallocatedResources() {
+ // unallocated resources = node capacity - containers allocation
+ // = -(container allocation - node capacity)
+ ResourceUtilization allocationClone =
+ ResourceUtilization.newInstance(containersAllocation);
+ getContainersMonitor()
+ .subtractNodeResourcesFromResourceUtilization(allocationClone);
+
+ Resource unallocated = UNAVAILABLE;
+ if (allocationClone.getCPU() <= 0 &&
+ allocationClone.getPhysicalMemory() <= 0 &&
+ allocationClone.getVirtualMemory() <= 0) {
+ int cpu = Math.round(allocationClone.getCPU() *
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ long memory = allocationClone.getPhysicalMemory();
+ unallocated = Resource.newInstance(-memory, -cpu);
+ }
+ return unallocated;
+ }
+
+
+ @Override
+ public Resource getAvailableResources() {
+ return getUnallocatedResources();
+ }
+
+ /**
+ * Add Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerLaunched(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerReleased(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 6e2b617..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the {@link ResourceUtilizationTracker} that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceUtilizationTracker implements
- ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
- AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to a container.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Add Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void addContainerResources(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void subtractContainerResource(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Check if NM has resources available currently to run the container.
- * @param container Container.
- * @return True, if NM has resources available currently to run the container.
- */
- @Override
- public boolean hasResourcesAvailable(Container container) {
- long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
- return hasResourcesAvailable(pMemBytes,
- (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
- container.getResource().getVirtualCores());
- }
-
- private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
- int cpuVcores) {
- // Check physical memory.
- if (LOG.isDebugEnabled()) {
- LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
- this.containersAllocation.getPhysicalMemory(),
- (pMemBytes >> 20),
- (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
- }
- if (this.containersAllocation.getPhysicalMemory() +
- (int) (pMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getPmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before vMemCheck" +
- "[isEnabled={}, current={} + asked={} > allowed={}]",
- getContainersMonitor().isVmemCheckEnabled(),
- this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
- (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
- }
- // Check virtual memory.
- if (getContainersMonitor().isVmemCheckEnabled() &&
- this.containersAllocation.getVirtualMemory() +
- (int) (vMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getVmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before cpuCheck [asked={} > allowed={}]",
- this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers());
- }
- // Check CPU. Compare using integral values of cores to avoid decimal
- // inaccuracies.
- if (!hasEnoughCpu(this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
- return false;
- }
- return true;
- }
-
- /**
- * Returns whether there is enough space for coresRequested in totalCores.
- * Converts currentAllocation usage to nearest integer count before comparing,
- * as floats are inherently imprecise. NOTE: this calculation assumes that
- * requested core counts must be integers, and currentAllocation core count
- * must also be an integer.
- *
- * @param currentAllocation The current allocation, a float value from 0 to 1.
- * @param totalCores The total cores in the system.
- * @param coresRequested The number of cores requested.
- * @return True if currentAllocationtotalCores*coresRequested <=
- * totalCores.
- */
- public boolean hasEnoughCpu(float currentAllocation, long totalCores,
- int coresRequested) {
- // Must not cast here, as it would truncate the decimal digits.
- return Math.round(currentAllocation * totalCores)
- + coresRequested <= totalCores;
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index a61b9d1..0bebe44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,10 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
+ // sum of the resources requested by guaranteed containers in queue
+ private final Resource guaranteedResourcesDemanded =
+ Resource.newInstance(0, 0);
+
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -88,6 +94,10 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
+ // sum of the resources to be released by opportunistic containers that
+ // have been marked to be killed or paused.
+ private final Resource opportunisticResourcesToBeReleased =
+ Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -128,6 +138,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
+ }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -155,20 +176,16 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ // We assume over allocation configurations have been initialized
+ this.utilizationTracker = getResourceTracker();
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.utilizationTracker =
- new AllocationBasedResourceUtilizationTracker(this);
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
+ private AllocationBasedResourceTracker getResourceTracker() {
+ if (context.isOverAllocationEnabled()) {
+ return new UtilizationBasedResourceTracker(this);
+ } else {
+ return new AllocationBasedResourceTracker(this);
+ }
}
/**
@@ -191,14 +208,18 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateContainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(maxOppQueueLength <= 0);
+ startPendingContainers(false);
+ break;
+ case SCHEDULE_CONTAINERS:
+ startPendingContainers(true);
+ break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -213,10 +234,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.subtractContainerResource(
+ this.utilizationTracker.containerReleased(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.addContainerResources(
+ this.utilizationTracker.containerLaunched(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -232,17 +253,20 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- //Kill/pause opportunistic containers if any to make room for
- // promotion request
- reclaimOpportunisticContainerResources(updateEvent.getContainer());
+ Resources.addTo(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
+ startPendingContainers(true);
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.subtractFrom(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
+ startPendingContainers(false);
}
}
try {
@@ -269,6 +293,7 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -279,7 +304,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.addContainerResources(container);
+ utilizationTracker.containerLaunched(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -344,65 +369,107 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- oppContainersToKill.remove(container.getContainerId());
+ ContainerId containerId = container.getContainerId();
+
+ // This could be killed externally for eg. by the ContainerManager,
+ // in which case, the container might still be queued.
+ if (queuedOpportunisticContainers.remove(containerId) != null) {
+ return;
+ }
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- Container queued =
- queuedOpportunisticContainers.remove(container.getContainerId());
- if (queued == null) {
- queuedGuaranteedContainers.remove(container.getContainerId());
+ if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ return;
+ }
+
+ if (oppContainersToKill.remove(containerId) != null) {
+ Resources.subtractFrom(
+ opportunisticResourcesToBeReleased, container.getResource());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(container.getContainerId(), container);
+ queuedGuaranteedContainers.put(containerId, container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else {
- queuedOpportunisticContainers.put(
- container.getContainerId(), container);
+ queuedOpportunisticContainers.put(containerId, container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(container
- .getContainerId());
+ Container completedContainer = runningContainers.remove(containerId);
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.subtractContainerResource(container);
+ this.utilizationTracker.containerReleased(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
+
+ // In case of over-allocation being turned on, we may need to reclaim
+ // more resources since the opportunistic containers that have been
+ // killed or paused may have not released as much resource as we need.
+ boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
+ startPendingContainers(reclaimOpportunisticResources);
}
}
/**
* Start pending containers in the queue.
- * @param forceStartGuaranteedContaieners When this is true, start guaranteed
- * container without looking at available resource
+ * @param reclaimOpportunisticResources if set to true, resources allocated
+ * to running OPPORTUNISTIC containers will be reclaimed in
+ * cases where there are GUARANTEED containers being queued
*/
- private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
- // Start guaranteed containers that are paused, if resources available.
- boolean resourcesAvailable = startContainers(
- queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
- // Start opportunistic containers, if resources available.
- if (resourcesAvailable) {
- startContainers(queuedOpportunisticContainers.values(), false);
+ private void startPendingContainers(boolean reclaimOpportunisticResources) {
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources and
+ // skip scanning the queue of opportunistic containers
+ if (maxOppQueueLength <= 0) {
+ forcefullyStartGuaranteedContainers();
+ return;
+ }
+
+ Resource available = utilizationTracker.getAvailableResources();
+
+ // Start guaranteed containers that are queued, if resources available.
+ boolean allGuaranteedContainersLaunched =
+ startGuaranteedContainers(available);
+ // Start opportunistic containers, if resources available, which is true
+ // if all guaranteed containers in queue have been launched.
+ if (allGuaranteedContainersLaunched) {
+ startOpportunisticContainers(available);
+ } else {
+ // If not all guaranteed containers in queue are launched, we may need
+ // to reclaim resources from opportunistic containers that are running.
+ if (reclaimOpportunisticResources) {
+ reclaimOpportunisticContainerResources();
+ }
}
}
- private boolean startContainers(
- Collection<Container> containersToBeStarted, boolean force) {
- Iterator<Container> cIter = containersToBeStarted.iterator();
+ /**
+ * Try to launch as many GUARANTEED containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all queued GUARANTEED containers are launched
+ * or there is no GUARANTEED containers to launch
+ */
+ private boolean startGuaranteedContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (tryStartContainer(container, force)) {
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -410,25 +477,49 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- private boolean tryStartContainer(Container container, boolean force) {
- boolean containerStarted = false;
- // call startContainer without checking available resource when force==true
- if (force || resourceAvailableToStartContainer(
- container)) {
+ /**
+ * Launch all queued GUARANTEED containers without checking resource
+ * availability. This is an optimization in cases where OPPORTUNISTIC
+ * containers are not allowed on the node.
+ */
+ private void forcefullyStartGuaranteedContainers() {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
+ while (cIter.hasNext()) {
+ Container container = cIter.next();
startContainer(container);
- containerStarted = true;
+ cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
}
- return containerStarted;
}
-
/**
- * Check if there is resource available to start a given container
- * immediately. (This can be extended to include overallocated resources)
- * @param container the container to start
- * @return true if container can be launched directly
+ * Try to launch as many OPPORTUNISTIC containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all OPPORTUNISTIC containers are launched
+ * or there is no OPPORTUNISTIC containers to launch
*/
- private boolean resourceAvailableToStartContainer(Container container) {
- return this.utilizationTracker.hasResourcesAvailable(container);
+ private boolean startOpportunisticContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedOpportunisticContainers.values().iterator();
+ boolean resourcesAvailable = true;
+ while (cIter.hasNext() && resourcesAvailable) {
+ Container container = cIter.next();
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
+ cIter.remove();
+ } else {
+ resourcesAvailable = false;
+ }
+ }
+ return resourcesAvailable;
+ }
+
+ private static boolean isResourceAvailable(
+ Resource resource, Container container) {
+ Resource left = Resources.subtract(resource, container.getResource());
+ return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
}
private boolean enqueueContainer(Container container) {
@@ -438,6 +529,7 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -482,18 +574,7 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
-
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources.
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
-
- // if the guaranteed container is queued, we need to preempt opportunistic
- // containers for make room for it
- if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
- reclaimOpportunisticContainerResources(container);
- }
+ startPendingContainers(true);
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -511,19 +592,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources(Container container) {
+ private void reclaimOpportunisticContainerResources() {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources(
- container.getContainerId());
- // Kill the opportunistic containers that were chosen.
- for (Container contToReclaim : extraOppContainersToReclaim) {
+ pickOpportunisticContainersToReclaimResources();
+ killOpportunisticContainers(extraOppContainersToReclaim);
+ }
+
+ private void killOpportunisticContainers(
+ Collection<Container> containersToReclaim) {
+ for (Container contToReclaim : containersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "resumed";
- LOG.info(
- "Container {} will be {} to start the "
- + "execution of guaranteed container {}.",
- contToReclaim.getContainerId(), preemptionAction,
- container.getContainerId());
+ "preempted";
+ LOG.info("Container {} will be {} to start the execution of guaranteed" +
+ " containers.", contToReclaim.getContainerId(), preemptionAction);
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -534,6 +615,8 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
+ Resources.addTo(
+ opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -542,7 +625,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.addContainerResources(container);
+ this.utilizationTracker.containerLaunched(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -551,14 +634,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources(
- ContainerId containerToStartId) {
+ private List<Container> pickOpportunisticContainersToReclaimResources() {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
- containerToStartId);
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -577,15 +658,19 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
+ // In the case of over-allocation, the running container may not
+ // release as much resources as it has requested, but we'll check
+ // again if more containers need to be killed/paused when this
+ // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
- "at the moment. Opportunistic containers are in the process of" +
- "being killed to make room.", containerToStartId);
+ LOG.warn("There are no sufficient resources to start guaranteed" +
+ " containers at the moment. Opportunistic containers are in" +
+ " the process of being killed to make room.");
}
return extraOpportContainersToKill;
}
@@ -600,34 +685,42 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- private ResourceUtilization resourcesToFreeUp(
- ContainerId containerToStartId) {
+ /**
+ * Determine how much resources are needed to be freed up to launch the given
+ * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
+ * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
+ * be killed/paused will release the amount of resources they have requested.
+ *
+ * If the node is over-allocating itself, this may cause not enough
+ * OPPORTUNISTIC containers being killed/paused in cases where the running
+ * OPPORTUNISTIC containers are not consuming fully their resource requests.
+ * We'd check again upon container completion events to see if more running
+ * OPPORTUNISTIC containers need to be killed/paused.
+ *
+ * @return the amount of resource needed to be reclaimed for this container
+ */
+ private ResourceUtilization resourcesToFreeUp() {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(this.utilizationTracker.getCurrentUtilization());
-
- // Add to the allocation the allocation of the pending guaranteed
- // containers that will start before the current container will be started.
- for (Container container : queuedGuaranteedContainers.values()) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
- if (container.getContainerId().equals(containerToStartId)) {
- break;
- }
- }
+ .newInstance(0, 0, 0.0f);
+
+ // Add to the allocation the allocation of pending guaranteed containers.
+ ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, guaranteedResourcesDemanded);
// These resources are being freed, likely at the behest of another
// guaranteed container..
- for (Container container : oppContainersToKill.values()) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
+
+ // Deduct any remaining resources available
+ Resource availableResources = utilizationTracker.getAvailableResources();
+ if (availableResources.getVirtualCores() > 0 &&
+ availableResources.getMemorySize() > 0) {
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, availableResources);
}
- // Subtract the overall node resources.
- getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
- resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 294eddf..9ad4f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,5 +28,7 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED
+ RECOVERY_COMPLETED,
+ // Producer: Containers Monitor when over-allocation is on
+ SCHEDULE_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
new file mode 100644
index 0000000..58b73d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how much
+ * resources are available to launch containers when over-allocation is on.
+ */
+public abstract class NMAllocationPolicy {
+ protected final ResourceThresholds overAllocationThresholds;
+ protected final ContainersMonitor containersMonitor;
+
+ public NMAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationThresholds = overAllocationThresholds;
+ }
+
+ /**
+ * Handle container launch events.
+ * @param container the container that has been launched
+ */
+ public void containerLaunched(Container container) {
+
+ }
+
+ /**
+ * Handle container release events.
+ * @param container the container that has been released
+ */
+ public void containerReleased(Container container) {
+
+ }
+
+ /**
+ * Get the amount of resources to launch containers when
+ * over-allocation is turned on.
+ * @return the amount of resources available to launch containers
+ */
+ public abstract Resource getAvailableResources();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 3c17eca..98d99c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -38,22 +39,20 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Add Container's resources to Node Utilization.
- * @param container Container.
+ * Get the amount of resources currently available to launch containers.
+ * @return Resource resources available to launch containers
*/
- void addContainerResources(Container container);
+ Resource getAvailableResources();
/**
- * Subtract Container's resources to Node Utilization.
+ * Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
- void subtractContainerResource(Container container);
+ void containerLaunched(Container container);
/**
- * Check if NM has resources available currently to run the container.
+ * Subtract Container's resources to Node Utilization upon container release.
* @param container Container.
- * @return True, if NM has resources available currently to run the container.
*/
- boolean hasResourcesAvailable(Container container);
-
+ void containerReleased(Container container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
new file mode 100644
index 0000000..f486506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of NMAllocationPolicy based on the
+ * snapshot of the latest containers utilization to determine how much
+ * resources are available * to launch containers when over-allocation
+ * is turned on.
+ */
+public class SnapshotBasedOverAllocationPolicy
+ extends NMAllocationPolicy {
+
+ public SnapshotBasedOverAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ super(overAllocationThresholds, containersMonitor);
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ ResourceUtilization utilization =
+ containersMonitor.getContainersUtilization(true).getUtilization();
+ long memoryAvailable = Math.round(
+ overAllocationThresholds.getMemoryThreshold() *
+ containersMonitor.getPmemAllocatedForContainers()) -
+ (utilization.getPhysicalMemory() << 20);
+ int vcoreAvailable = Math.round(
+ (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
+ containersMonitor.getVCoresAllocatedForContainers());
+ return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
new file mode 100644
index 0000000..6f9bc82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+* An resource availability tracker that determines if there are resources
+* available based on if there are unallocated resources or if there are
+* un-utilized resources.
+*/
+public class UtilizationBasedResourceTracker
+ extends AllocationBasedResourceTracker {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private final NMAllocationPolicy overAllocationPolicy;
+
+ UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
+ super(scheduler);
+ this.overAllocationPolicy =
+ getContainersMonitor().getContainerOverAllocationPolicy();
+ }
+
+ @Override
+ public void containerLaunched(Container container) {
+ super.containerLaunched(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerLaunched(container);
+ }
+ }
+
+ @Override
+ public void containerReleased(Container container) {
+ super.containerReleased(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerReleased(container);
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnUtilization =
+ getResourcesAvailableBasedOnUtilization();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The amount of resources available based on allocation is " +
+ resourceBasedOnAllocation + ", based on utilization is " +
+ resourceBasedOnUtilization);
+ }
+
+ return Resources.componentwiseMax(resourceBasedOnAllocation,
+ resourceBasedOnUtilization);
+ }
+
+ /**
+ * Get the amount of resources based on the slack between
+ * the actual utilization and desired utilization.
+ * @return Resource resource available
+ */
+ private Resource getResourcesAvailableBasedOnUtilization() {
+ if (overAllocationPolicy == null) {
+ return Resources.none();
+ }
+
+ return overAllocationPolicy.getAvailableResources();
+ }
+
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return getContainersMonitor().getContainersUtilization(false)
+ .getUtilization();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 493aa4c..92613ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -345,6 +346,40 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ ContainerSubState finalState)
+ throws InterruptedException, YarnException, IOException {
+ waitForContainerSubState(containerManager, containerID,
+ Arrays.asList(finalState), 20);
+ }
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ List<ContainerSubState> finalStates, int timeOutMax)
+ throws InterruptedException, YarnException, IOException {
+ List<ContainerId> list = new ArrayList<>();
+ list.add(containerID);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(list);
+ ContainerStatus containerStatus;
+ HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
+ int timeoutSecs = 0;
+ do {
+ Thread.sleep(1000);
+ containerStatus =
+ containerManager.getContainerStatuses(request)
+ .getContainerStatuses().get(0);
+ LOG.info("Waiting for container to get into one of states " + fStates
+ + ". Current state is " + containerStatus.getContainerSubState());
+ timeoutSecs += 1;
+ } while (!fStates.contains(containerStatus.getContainerSubState())
+ && timeoutSecs < timeOutMax);
+ LOG.info("Container state is " + containerStatus.getContainerSubState());
+ Assert.assertTrue("ContainerSubState is not correct (timedout)",
+ fStates.contains(containerStatus.getContainerSubState()));
+ }
+
+
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 8aee532..c071283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization()
+ 0, containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,8 +303,9 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization()
- .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization(false)
+ .getUtilization().compareTo(
+ ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -316,7 +317,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization()
+ 0 != containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
new file mode 100644
index 0000000..1e8bfdf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceTracker} class.
+ */
+public class TestAllocationBasedResourceTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceTracker tracker =
+ new AllocationBasedResourceTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ tracker.containerLaunched(testContainer);
+ }
+ Assert.assertFalse(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ }
+
+ private static boolean isResourcesAvailable(
+ Resource available, Container container) {
+ return available.compareTo(container.getResource()) >= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 82c2147..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
- */
-public class TestAllocationBasedResourceUtilizationTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
- tracker.addContainerResources(testContainer);
- }
- Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
- }
-
- /**
- * Test the case where the current allocation has been truncated to 0.8888891
- * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
- * true.
- */
- @Test
- public void testHasEnoughCpu() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- float currentAllocation = 0.8888891f;
- long totalCores = 9;
- int alreadyUsedCores = 8;
- Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores));
- Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores + 1));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/24] hadoop git commit: YARN-6675. Add NM support to launch
opportunistic containers based on overallocation. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/879a5e7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/879a5e7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/879a5e7b
Branch: refs/heads/YARN-1011
Commit: 879a5e7b3a89e34305c699e52a8e5da288505c6c
Parents: 0456591
Author: Miklos Szegedi <sz...@apache.org>
Authored: Fri Apr 20 14:15:29 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:14:58 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 ++
...locationBasedResourceUtilizationTracker.java | 158 ---
.../scheduler/ContainerScheduler.java | 317 +++--
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 +
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 +
.../UtilizationBasedResourceTracker.java | 95 ++
.../BaseContainerManagerTest.java | 35 +
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 ++
...locationBasedResourceUtilizationTracker.java | 93 --
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ++++++++++++++++++
20 files changed, 1916 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 572684e..d757376 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization();
+ return containersMonitor.getContainersUtilization(false).getUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 27a7c80..a08e227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,6 +245,12 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
+
+ // ContainersLauncher must be added after ContainersMonitor
+ // because the former depends on the latter to initialize
+ // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -269,8 +275,6 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..0228332 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- private String getContainerPid(Path pidFilePath) throws Exception {
+ protected String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 7870f86..2f5acfa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,8 +121,7 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
- event.getContainer(), dirsHandler, containerManager);
+ createContainerLaunch(app, event.getContainer());
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -225,4 +224,10 @@ public class ContainersLauncher extends AbstractService
break;
}
}
+
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 64831e9..8da4ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,10 +23,24 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
- ResourceUtilization getContainersUtilization();
+
+ /**
+ * Get the aggregate resource utilization of containers running on the node,
+ * with a timestamp of the measurement.
+ * @param latest true if the latest result should be returned
+ * @return ResourceUtilization resource utilization of all containers
+ */
+ ContainersResourceUtilization getContainersUtilization(boolean latest);
+
+ /**
+ * Get the policy to over-allocate containers when over-allocation is on.
+ * @return null if over-allocation is turned off
+ */
+ NMAllocationPolicy getContainerOverAllocationPolicy();
float getVmemRatio();
@@ -66,4 +80,26 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
+
+ /**
+ * A snapshot of resource utilization of all containers with the timestamp.
+ */
+ final class ContainersResourceUtilization {
+ private final ResourceUtilization utilization;
+ private final long timestamp;
+
+ public ContainersResourceUtilization(
+ ResourceUtilization utilization, long timestamp) {
+ this.utilization = utilization;
+ this.timestamp = timestamp;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public ResourceUtilization getUtilization() {
+ return utilization;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 7873882..a045d78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,8 +116,9 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ResourceUtilization containersUtilization;
+ private ContainersResourceUtilization latestContainersUtilization;
+ private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -129,7 +134,8 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
}
@Override
@@ -365,6 +371,10 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
+ // TODO make this configurable
+ this.overAllocationPolicy =
+ createOverAllocationPolicy(resourceThresholds);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -372,6 +382,11 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
+ protected NMAllocationPolicy createOverAllocationPolicy(
+ ResourceThresholds resourceThresholds) {
+ return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -655,7 +670,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setContainersUtilization(trackedContainersUtilization);
+ setLatestContainersUtilization(trackedContainersUtilization);
+
+ // check opportunity to start containers if over-allocation is on
+ if (context.isOverAllocationEnabled()) {
+ attemptToStartContainersUponLowUtilization();
+ }
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1045,12 +1065,34 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ResourceUtilization getContainersUtilization() {
- return this.containersUtilization;
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ // TODO update containerUtilization if latest is true
+ return this.latestContainersUtilization;
+ }
+
+ @Override
+ public NMAllocationPolicy getContainerOverAllocationPolicy() {
+ return overAllocationPolicy;
+ }
+
+ private void setLatestContainersUtilization(ResourceUtilization utilization) {
+ this.latestContainersUtilization = new ContainersResourceUtilization(
+ utilization, System.currentTimeMillis());
}
- private void setContainersUtilization(ResourceUtilization utilization) {
- this.containersUtilization = utilization;
+ @VisibleForTesting
+ public void attemptToStartContainersUponLowUtilization() {
+ if (getContainerOverAllocationPolicy() != null) {
+ Resource available = getContainerOverAllocationPolicy()
+ .getAvailableResources();
+ if (available.getMemorySize() > 0 &&
+ available.getVirtualCores() > 0) {
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(null,
+ ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
new file mode 100644
index 0000000..86b3698
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the resource utilization tracker that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceTracker
+ implements ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private static final Resource UNAVAILABLE =
+ Resource.newInstance(0, 0);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+
+ AllocationBasedResourceTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to containers.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Get the amount of resources that have not been allocated to containers.
+ * @return Resource resources that have not been allocated to containers.
+ */
+ protected Resource getUnallocatedResources() {
+ // unallocated resources = node capacity - containers allocation
+ // = -(container allocation - node capacity)
+ ResourceUtilization allocationClone =
+ ResourceUtilization.newInstance(containersAllocation);
+ getContainersMonitor()
+ .subtractNodeResourcesFromResourceUtilization(allocationClone);
+
+ Resource unallocated = UNAVAILABLE;
+ if (allocationClone.getCPU() <= 0 &&
+ allocationClone.getPhysicalMemory() <= 0 &&
+ allocationClone.getVirtualMemory() <= 0) {
+ int cpu = Math.round(allocationClone.getCPU() *
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ long memory = allocationClone.getPhysicalMemory();
+ unallocated = Resource.newInstance(-memory, -cpu);
+ }
+ return unallocated;
+ }
+
+
+ @Override
+ public Resource getAvailableResources() {
+ return getUnallocatedResources();
+ }
+
+ /**
+ * Add Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerLaunched(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated allocation.
+ * @param container Container.
+ */
+ @Override
+ public void containerReleased(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 6e2b617..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the {@link ResourceUtilizationTracker} that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceUtilizationTracker implements
- ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
- AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to a container.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Add Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void addContainerResources(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated Utilization.
- * @param container Container.
- */
- @Override
- public void subtractContainerResource(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Check if NM has resources available currently to run the container.
- * @param container Container.
- * @return True, if NM has resources available currently to run the container.
- */
- @Override
- public boolean hasResourcesAvailable(Container container) {
- long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
- return hasResourcesAvailable(pMemBytes,
- (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
- container.getResource().getVirtualCores());
- }
-
- private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
- int cpuVcores) {
- // Check physical memory.
- if (LOG.isDebugEnabled()) {
- LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
- this.containersAllocation.getPhysicalMemory(),
- (pMemBytes >> 20),
- (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
- }
- if (this.containersAllocation.getPhysicalMemory() +
- (int) (pMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getPmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before vMemCheck" +
- "[isEnabled={}, current={} + asked={} > allowed={}]",
- getContainersMonitor().isVmemCheckEnabled(),
- this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
- (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
- }
- // Check virtual memory.
- if (getContainersMonitor().isVmemCheckEnabled() &&
- this.containersAllocation.getVirtualMemory() +
- (int) (vMemBytes >> 20) >
- (int) (getContainersMonitor()
- .getVmemAllocatedForContainers() >> 20)) {
- return false;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("before cpuCheck [asked={} > allowed={}]",
- this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers());
- }
- // Check CPU. Compare using integral values of cores to avoid decimal
- // inaccuracies.
- if (!hasEnoughCpu(this.containersAllocation.getCPU(),
- getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
- return false;
- }
- return true;
- }
-
- /**
- * Returns whether there is enough space for coresRequested in totalCores.
- * Converts currentAllocation usage to nearest integer count before comparing,
- * as floats are inherently imprecise. NOTE: this calculation assumes that
- * requested core counts must be integers, and currentAllocation core count
- * must also be an integer.
- *
- * @param currentAllocation The current allocation, a float value from 0 to 1.
- * @param totalCores The total cores in the system.
- * @param coresRequested The number of cores requested.
- * @return True if currentAllocationtotalCores*coresRequested <=
- * totalCores.
- */
- public boolean hasEnoughCpu(float currentAllocation, long totalCores,
- int coresRequested) {
- // Must not cast here, as it would truncate the decimal digits.
- return Math.round(currentAllocation * totalCores)
- + coresRequested <= totalCores;
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index a61b9d1..0bebe44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,10 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
+ // sum of the resources requested by guaranteed containers in queue
+ private final Resource guaranteedResourcesDemanded =
+ Resource.newInstance(0, 0);
+
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -88,6 +94,10 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
+ // sum of the resources to be released by opportunistic containers that
+ // have been marked to be killed or paused.
+ private final Resource opportunisticResourcesToBeReleased =
+ Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -128,6 +138,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
+ }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -155,20 +176,16 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ // We assume over allocation configurations have been initialized
+ this.utilizationTracker = getResourceTracker();
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.utilizationTracker =
- new AllocationBasedResourceUtilizationTracker(this);
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
+ private AllocationBasedResourceTracker getResourceTracker() {
+ if (context.isOverAllocationEnabled()) {
+ return new UtilizationBasedResourceTracker(this);
+ } else {
+ return new AllocationBasedResourceTracker(this);
+ }
}
/**
@@ -191,14 +208,18 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateContainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(maxOppQueueLength <= 0);
+ startPendingContainers(false);
+ break;
+ case SCHEDULE_CONTAINERS:
+ startPendingContainers(true);
+ break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -213,10 +234,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.subtractContainerResource(
+ this.utilizationTracker.containerReleased(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.addContainerResources(
+ this.utilizationTracker.containerLaunched(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -232,17 +253,20 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- //Kill/pause opportunistic containers if any to make room for
- // promotion request
- reclaimOpportunisticContainerResources(updateEvent.getContainer());
+ Resources.addTo(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
+ startPendingContainers(true);
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.subtractFrom(guaranteedResourcesDemanded,
+ updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
+ startPendingContainers(false);
}
}
try {
@@ -269,6 +293,7 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -279,7 +304,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.addContainerResources(container);
+ utilizationTracker.containerLaunched(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -344,65 +369,107 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- oppContainersToKill.remove(container.getContainerId());
+ ContainerId containerId = container.getContainerId();
+
+ // This could be killed externally for eg. by the ContainerManager,
+ // in which case, the container might still be queued.
+ if (queuedOpportunisticContainers.remove(containerId) != null) {
+ return;
+ }
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- Container queued =
- queuedOpportunisticContainers.remove(container.getContainerId());
- if (queued == null) {
- queuedGuaranteedContainers.remove(container.getContainerId());
+ if (queuedGuaranteedContainers.remove(containerId) != null) {
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ return;
+ }
+
+ if (oppContainersToKill.remove(containerId) != null) {
+ Resources.subtractFrom(
+ opportunisticResourcesToBeReleased, container.getResource());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(container.getContainerId(), container);
+ queuedGuaranteedContainers.put(containerId, container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else {
- queuedOpportunisticContainers.put(
- container.getContainerId(), container);
+ queuedOpportunisticContainers.put(containerId, container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(container
- .getContainerId());
+ Container completedContainer = runningContainers.remove(containerId);
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.subtractContainerResource(container);
+ this.utilizationTracker.containerReleased(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
+
+ // In case of over-allocation being turned on, we may need to reclaim
+ // more resources since the opportunistic containers that have been
+ // killed or paused may have not released as much resource as we need.
+ boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
+ startPendingContainers(reclaimOpportunisticResources);
}
}
/**
* Start pending containers in the queue.
- * @param forceStartGuaranteedContaieners When this is true, start guaranteed
- * container without looking at available resource
+ * @param reclaimOpportunisticResources if set to true, resources allocated
+ * to running OPPORTUNISTIC containers will be reclaimed in
+ * cases where there are GUARANTEED containers being queued
*/
- private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
- // Start guaranteed containers that are paused, if resources available.
- boolean resourcesAvailable = startContainers(
- queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
- // Start opportunistic containers, if resources available.
- if (resourcesAvailable) {
- startContainers(queuedOpportunisticContainers.values(), false);
+ private void startPendingContainers(boolean reclaimOpportunisticResources) {
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources and
+ // skip scanning the queue of opportunistic containers
+ if (maxOppQueueLength <= 0) {
+ forcefullyStartGuaranteedContainers();
+ return;
+ }
+
+ Resource available = utilizationTracker.getAvailableResources();
+
+ // Start guaranteed containers that are queued, if resources available.
+ boolean allGuaranteedContainersLaunched =
+ startGuaranteedContainers(available);
+ // Start opportunistic containers, if resources available, which is true
+ // if all guaranteed containers in queue have been launched.
+ if (allGuaranteedContainersLaunched) {
+ startOpportunisticContainers(available);
+ } else {
+ // If not all guaranteed containers in queue are launched, we may need
+ // to reclaim resources from opportunistic containers that are running.
+ if (reclaimOpportunisticResources) {
+ reclaimOpportunisticContainerResources();
+ }
}
}
- private boolean startContainers(
- Collection<Container> containersToBeStarted, boolean force) {
- Iterator<Container> cIter = containersToBeStarted.iterator();
+ /**
+ * Try to launch as many GUARANTEED containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all queued GUARANTEED containers are launched
+ * or there is no GUARANTEED containers to launch
+ */
+ private boolean startGuaranteedContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (tryStartContainer(container, force)) {
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -410,25 +477,49 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- private boolean tryStartContainer(Container container, boolean force) {
- boolean containerStarted = false;
- // call startContainer without checking available resource when force==true
- if (force || resourceAvailableToStartContainer(
- container)) {
+ /**
+ * Launch all queued GUARANTEED containers without checking resource
+ * availability. This is an optimization in cases where OPPORTUNISTIC
+ * containers are not allowed on the node.
+ */
+ private void forcefullyStartGuaranteedContainers() {
+ Iterator<Container> cIter =
+ queuedGuaranteedContainers.values().iterator();
+ while (cIter.hasNext()) {
+ Container container = cIter.next();
startContainer(container);
- containerStarted = true;
+ cIter.remove();
+ Resources.subtractFrom(
+ guaranteedResourcesDemanded, container.getResource());
}
- return containerStarted;
}
-
/**
- * Check if there is resource available to start a given container
- * immediately. (This can be extended to include overallocated resources)
- * @param container the container to start
- * @return true if container can be launched directly
+ * Try to launch as many OPPORTUNISTIC containers as possible.
+ * @param available the amount of resources available to launch containers
+ * @return true if all OPPORTUNISTIC containers are launched
+ * or there is no OPPORTUNISTIC containers to launch
*/
- private boolean resourceAvailableToStartContainer(Container container) {
- return this.utilizationTracker.hasResourcesAvailable(container);
+ private boolean startOpportunisticContainers(Resource available) {
+ Iterator<Container> cIter =
+ queuedOpportunisticContainers.values().iterator();
+ boolean resourcesAvailable = true;
+ while (cIter.hasNext() && resourcesAvailable) {
+ Container container = cIter.next();
+ if (isResourceAvailable(available, container)) {
+ startContainer(container);
+ Resources.subtractFrom(available, container.getResource());
+ cIter.remove();
+ } else {
+ resourcesAvailable = false;
+ }
+ }
+ return resourcesAvailable;
+ }
+
+ private static boolean isResourceAvailable(
+ Resource resource, Container container) {
+ Resource left = Resources.subtract(resource, container.getResource());
+ return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
}
private boolean enqueueContainer(Container container) {
@@ -438,6 +529,7 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
+ Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -482,18 +574,7 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
-
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources.
- boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
- startPendingContainers(forceStartGuaranteedContainers);
-
- // if the guaranteed container is queued, we need to preempt opportunistic
- // containers for make room for it
- if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
- reclaimOpportunisticContainerResources(container);
- }
+ startPendingContainers(true);
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -511,19 +592,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources(Container container) {
+ private void reclaimOpportunisticContainerResources() {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources(
- container.getContainerId());
- // Kill the opportunistic containers that were chosen.
- for (Container contToReclaim : extraOppContainersToReclaim) {
+ pickOpportunisticContainersToReclaimResources();
+ killOpportunisticContainers(extraOppContainersToReclaim);
+ }
+
+ private void killOpportunisticContainers(
+ Collection<Container> containersToReclaim) {
+ for (Container contToReclaim : containersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "resumed";
- LOG.info(
- "Container {} will be {} to start the "
- + "execution of guaranteed container {}.",
- contToReclaim.getContainerId(), preemptionAction,
- container.getContainerId());
+ "preempted";
+ LOG.info("Container {} will be {} to start the execution of guaranteed" +
+ " containers.", contToReclaim.getContainerId(), preemptionAction);
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -534,6 +615,8 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
+ Resources.addTo(
+ opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -542,7 +625,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.addContainerResources(container);
+ this.utilizationTracker.containerLaunched(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -551,14 +634,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources(
- ContainerId containerToStartId) {
+ private List<Container> pickOpportunisticContainersToReclaimResources() {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
- containerToStartId);
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -577,15 +658,19 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
+ // In the case of over-allocation, the running container may not
+ // release as much resources as it has requested, but we'll check
+ // again if more containers need to be killed/paused when this
+ // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
- "at the moment. Opportunistic containers are in the process of" +
- "being killed to make room.", containerToStartId);
+ LOG.warn("There are no sufficient resources to start guaranteed" +
+ " containers at the moment. Opportunistic containers are in" +
+ " the process of being killed to make room.");
}
return extraOpportContainersToKill;
}
@@ -600,34 +685,42 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- private ResourceUtilization resourcesToFreeUp(
- ContainerId containerToStartId) {
+ /**
+ * Determine how much resources are needed to be freed up to launch the given
+ * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
+ * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
+ * be killed/paused will release the amount of resources they have requested.
+ *
+ * If the node is over-allocating itself, this may cause not enough
+ * OPPORTUNISTIC containers being killed/paused in cases where the running
+ * OPPORTUNISTIC containers are not consuming fully their resource requests.
+ * We'd check again upon container completion events to see if more running
+ * OPPORTUNISTIC containers need to be killed/paused.
+ *
+ * @return the amount of resource needed to be reclaimed for this container
+ */
+ private ResourceUtilization resourcesToFreeUp() {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(this.utilizationTracker.getCurrentUtilization());
-
- // Add to the allocation the allocation of the pending guaranteed
- // containers that will start before the current container will be started.
- for (Container container : queuedGuaranteedContainers.values()) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
- if (container.getContainerId().equals(containerToStartId)) {
- break;
- }
- }
+ .newInstance(0, 0, 0.0f);
+
+ // Add to the allocation the allocation of pending guaranteed containers.
+ ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, guaranteedResourcesDemanded);
// These resources are being freed, likely at the behest of another
// guaranteed container..
- for (Container container : oppContainersToKill.values()) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), resourceAllocationToFreeUp,
- container.getResource());
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
+
+ // Deduct any remaining resources available
+ Resource availableResources = utilizationTracker.getAvailableResources();
+ if (availableResources.getVirtualCores() > 0 &&
+ availableResources.getMemorySize() > 0) {
+ ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
+ resourceAllocationToFreeUp, availableResources);
}
- // Subtract the overall node resources.
- getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
- resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 294eddf..9ad4f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,5 +28,7 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED
+ RECOVERY_COMPLETED,
+ // Producer: Containers Monitor when over-allocation is on
+ SCHEDULE_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
new file mode 100644
index 0000000..58b73d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how much
+ * resources are available to launch containers when over-allocation is on.
+ */
+public abstract class NMAllocationPolicy {
+ protected final ResourceThresholds overAllocationThresholds;
+ protected final ContainersMonitor containersMonitor;
+
+ public NMAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationThresholds = overAllocationThresholds;
+ }
+
+ /**
+ * Handle container launch events.
+ * @param container the container that has been launched
+ */
+ public void containerLaunched(Container container) {
+
+ }
+
+ /**
+ * Handle container release events.
+ * @param container the container that has been released
+ */
+ public void containerReleased(Container container) {
+
+ }
+
+ /**
+ * Get the amount of resources to launch containers when
+ * over-allocation is turned on.
+ * @return the amount of resources available to launch containers
+ */
+ public abstract Resource getAvailableResources();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 3c17eca..98d99c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -38,22 +39,20 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Add Container's resources to Node Utilization.
- * @param container Container.
+ * Get the amount of resources currently available to launch containers.
+ * @return Resource resources available to launch containers
*/
- void addContainerResources(Container container);
+ Resource getAvailableResources();
/**
- * Subtract Container's resources to Node Utilization.
+ * Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
- void subtractContainerResource(Container container);
+ void containerLaunched(Container container);
/**
- * Check if NM has resources available currently to run the container.
+ * Subtract Container's resources to Node Utilization upon container release.
* @param container Container.
- * @return True, if NM has resources available currently to run the container.
*/
- boolean hasResourcesAvailable(Container container);
-
+ void containerReleased(Container container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
new file mode 100644
index 0000000..f486506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of NMAllocationPolicy based on the
+ * snapshot of the latest containers utilization to determine how much
+ * resources are available * to launch containers when over-allocation
+ * is turned on.
+ */
+public class SnapshotBasedOverAllocationPolicy
+ extends NMAllocationPolicy {
+
+ public SnapshotBasedOverAllocationPolicy(
+ ResourceThresholds overAllocationThresholds,
+ ContainersMonitor containersMonitor) {
+ super(overAllocationThresholds, containersMonitor);
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ ResourceUtilization utilization =
+ containersMonitor.getContainersUtilization(true).getUtilization();
+ long memoryAvailable = Math.round(
+ overAllocationThresholds.getMemoryThreshold() *
+ containersMonitor.getPmemAllocatedForContainers()) -
+ (utilization.getPhysicalMemory() << 20);
+ int vcoreAvailable = Math.round(
+ (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
+ containersMonitor.getVCoresAllocatedForContainers());
+ return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
new file mode 100644
index 0000000..6f9bc82
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+* An resource availability tracker that determines if there are resources
+* available based on if there are unallocated resources or if there are
+* un-utilized resources.
+*/
+public class UtilizationBasedResourceTracker
+ extends AllocationBasedResourceTracker {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
+
+ private final NMAllocationPolicy overAllocationPolicy;
+
+ UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
+ super(scheduler);
+ this.overAllocationPolicy =
+ getContainersMonitor().getContainerOverAllocationPolicy();
+ }
+
+ @Override
+ public void containerLaunched(Container container) {
+ super.containerLaunched(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerLaunched(container);
+ }
+ }
+
+ @Override
+ public void containerReleased(Container container) {
+ super.containerReleased(container);
+ if (overAllocationPolicy != null) {
+ overAllocationPolicy.containerReleased(container);
+ }
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnUtilization =
+ getResourcesAvailableBasedOnUtilization();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The amount of resources available based on allocation is " +
+ resourceBasedOnAllocation + ", based on utilization is " +
+ resourceBasedOnUtilization);
+ }
+
+ return Resources.componentwiseMax(resourceBasedOnAllocation,
+ resourceBasedOnUtilization);
+ }
+
+ /**
+ * Get the amount of resources based on the slack between
+ * the actual utilization and desired utilization.
+ * @return Resource resource available
+ */
+ private Resource getResourcesAvailableBasedOnUtilization() {
+ if (overAllocationPolicy == null) {
+ return Resources.none();
+ }
+
+ return overAllocationPolicy.getAvailableResources();
+ }
+
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return getContainersMonitor().getContainersUtilization(false)
+ .getUtilization();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 493aa4c..92613ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -345,6 +346,40 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ ContainerSubState finalState)
+ throws InterruptedException, YarnException, IOException {
+ waitForContainerSubState(containerManager, containerID,
+ Arrays.asList(finalState), 20);
+ }
+ public static void waitForContainerSubState(
+ ContainerManagementProtocol containerManager, ContainerId containerID,
+ List<ContainerSubState> finalStates, int timeOutMax)
+ throws InterruptedException, YarnException, IOException {
+ List<ContainerId> list = new ArrayList<>();
+ list.add(containerID);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(list);
+ ContainerStatus containerStatus;
+ HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
+ int timeoutSecs = 0;
+ do {
+ Thread.sleep(1000);
+ containerStatus =
+ containerManager.getContainerStatuses(request)
+ .getContainerStatuses().get(0);
+ LOG.info("Waiting for container to get into one of states " + fStates
+ + ". Current state is " + containerStatus.getContainerSubState());
+ timeoutSecs += 1;
+ } while (!fStates.contains(containerStatus.getContainerSubState())
+ && timeoutSecs < timeOutMax);
+ LOG.info("Container state is " + containerStatus.getContainerSubState());
+ Assert.assertTrue("ContainerSubState is not correct (timedout)",
+ fStates.contains(containerStatus.getContainerSubState()));
+ }
+
+
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 8aee532..c071283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization()
+ 0, containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,8 +303,9 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization()
- .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization(false)
+ .getUtilization().compareTo(
+ ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -316,7 +317,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization()
+ 0 != containersMonitor.getContainersUtilization(false).getUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
new file mode 100644
index 0000000..1e8bfdf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceTracker} class.
+ */
+public class TestAllocationBasedResourceTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceTracker tracker =
+ new AllocationBasedResourceTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ tracker.containerLaunched(testContainer);
+ }
+ Assert.assertFalse(
+ isResourcesAvailable(tracker.getAvailableResources(), testContainer));
+ }
+
+ private static boolean isResourcesAvailable(
+ Resource available, Container container) {
+ return available.compareTo(container.getResource()) >= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
deleted file mode 100644
index 82c2147..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
- */
-public class TestAllocationBasedResourceUtilizationTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
- tracker.addContainerResources(testContainer);
- }
- Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
- }
-
- /**
- * Test the case where the current allocation has been truncated to 0.8888891
- * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
- * true.
- */
- @Test
- public void testHasEnoughCpu() {
- AllocationBasedResourceUtilizationTracker tracker =
- new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
- float currentAllocation = 0.8888891f;
- long totalCores = 9;
- int alreadyUsedCores = 8;
- Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores));
- Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
- (int) totalCores - alreadyUsedCores + 1));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/24] hadoop git commit: YARN-1015. FS should watch node resource
utilization and allocate opportunistic containers if appropriate.
Posted by ha...@apache.org.
YARN-1015. FS should watch node resource utilization and allocate opportunistic containers if appropriate.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86555164
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86555164
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86555164
Branch: refs/heads/YARN-1011
Commit: 86555164c3e11275f5e8af2ecb208b5c0d444fb0
Parents: ce4c4a7
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Nov 17 07:47:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:02 2018 -0700
----------------------------------------------------------------------
.../sls/scheduler/FairSchedulerMetrics.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 13 +
.../scheduler/SchedulerNode.java | 48 ++
.../scheduler/fair/FSAppAttempt.java | 175 ++++---
.../scheduler/fair/FSLeafQueue.java | 50 +-
.../scheduler/fair/FSParentQueue.java | 36 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 39 +-
.../scheduler/fair/FairScheduler.java | 105 +++--
.../fair/FairSchedulerConfiguration.java | 5 +
.../scheduler/fair/Schedulable.java | 17 +-
.../DominantResourceFairnessPolicy.java | 8 +-
.../fair/policies/FairSharePolicy.java | 4 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 60 ++-
.../TestWorkPreservingRMRestart.java | 2 +-
.../scheduler/fair/FakeSchedulable.java | 9 +-
.../scheduler/fair/TestAppRunnability.java | 9 +-
.../scheduler/fair/TestFSAppAttempt.java | 4 +-
.../scheduler/fair/TestFSLeafQueue.java | 4 +-
.../scheduler/fair/TestFSSchedulerNode.java | 4 +-
.../scheduler/fair/TestFairScheduler.java | 468 +++++++++++++++++--
.../scheduler/fair/TestSchedulingPolicy.java | 10 +-
...ervicesFairSchedulerCustomResourceTypes.java | 2 +-
24 files changed, 869 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index a5aee74..1f4e7c7 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -75,7 +75,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getMemorySize();
case USAGE:
- return schedulable.getResourceUsage().getMemorySize();
+ return schedulable.getGuaranteedResourceUsage().getMemorySize();
case MINSHARE:
return schedulable.getMinShare().getMemorySize();
case MAXSHARE:
@@ -96,7 +96,7 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
case DEMAND:
return schedulable.getDemand().getVirtualCores();
case USAGE:
- return schedulable.getResourceUsage().getVirtualCores();
+ return schedulable.getGuaranteedResourceUsage().getVirtualCores();
case MINSHARE:
return schedulable.getMinShare().getVirtualCores();
case MAXSHARE:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 18ff086..71651a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -278,6 +278,11 @@ public class YarnConfiguration extends Configuration {
public static final String APP_NAME_PLACEMENT_RULE = "app-name";
+ public static final String RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED =
+ RM_PREFIX + "scheduler.oversubscription.enabled";
+ public static final boolean DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED
+ = false;
+
/** Enable Resource Manager webapp ui actions */
public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
RM_PREFIX + "webapp.ui-actions.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 985936a..b45fe06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -429,6 +429,19 @@
</property>
<property>
+ <description>
+ If set to true, the scheduler will try to over-allocate resources on the
+ nodes that allow overallocation. To enable overallocatin on a node, set
+ {code}yarn.nodemanager.overallocation.memory-utilization-threshold{code}
+ and
+ {code}yarn.nodemanager.overallocation.cpu-utilization-threshold{code}
+ to a number in the range (0.0, 1.0)
+ </description>
+ <name>yarn.resourcemanager.scheduler.oversubscription.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified. </description>
<name>yarn.resourcemanager.recovery.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 8f5ae0e..ff3640c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -62,6 +64,10 @@ public abstract class SchedulerNode {
private static final Log LOG = LogFactory.getLog(SchedulerNode.class);
private Resource capacity;
+ // The resource available within the node's capacity that can be given out
+ // to run GUARANTEED containers, including reserved, preempted and any
+ // remaining free resources. Resources allocated to OPPORTUNISTIC containers
+ // are tracked in allocatedResourceOpportunistic
private Resource unallocatedResource = Resource.newInstance(0, 0);
private RMContainer reservedContainer;
@@ -663,6 +669,48 @@ public abstract class SchedulerNode {
this.nodeAttributes = attributes;
}
+ /**
+ * Get the amount of resources that can be allocated to opportunistic
+ * containers in the case of overallocation. It is calculated as
+ * node capacity - (node utilization + resources of allocated-yet-not-started
+ * containers).
+ * @return the amount of resources that are available to be allocated to
+ * opportunistic containers
+ */
+ public synchronized Resource allowedResourceForOverAllocation() {
+ OverAllocationInfo overAllocationInfo = rmNode.getOverAllocationInfo();
+ if (overAllocationInfo == null) {
+ LOG.debug("Overallocation is disabled on node: " + rmNode.getHostName());
+ return Resources.none();
+ }
+
+ ResourceUtilization projectedNodeUtilization = ResourceUtilization.
+ newInstance(getNodeUtilization());
+ // account for resources allocated in this heartbeat
+ projectedNodeUtilization.addTo(
+ (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0,
+ (float) resourceAllocatedPendingLaunch.getVirtualCores() /
+ capacity.getVirtualCores());
+
+ ResourceThresholds thresholds =
+ overAllocationInfo.getOverAllocationThresholds();
+ Resource overAllocationThreshold = Resources.createResource(
+ (long) (capacity.getMemorySize() * thresholds.getMemoryThreshold()),
+ (int) (capacity.getVirtualCores() * thresholds.getCpuThreshold()));
+ long allowedMemory = Math.max(0, overAllocationThreshold.getMemorySize()
+ - projectedNodeUtilization.getPhysicalMemory());
+ int allowedCpu = Math.max(0, (int)
+ (overAllocationThreshold.getVirtualCores() -
+ projectedNodeUtilization.getCPU() * capacity.getVirtualCores()));
+
+ Resource resourceAllowedForOpportunisticContainers =
+ Resources.createResource(allowedMemory, allowedCpu);
+
+ // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
+ // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ return resourceAllowedForOpportunisticContainers;
+ }
+
private static class ContainerInfo {
private final RMContainer container;
private boolean launchedOnNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index d9f3262..1928591 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -169,7 +170,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedResource(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
@@ -178,30 +179,35 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
}
- private void unreserveInternal(
+ private boolean unreserveInternal(
SchedulerRequestKey schedulerKey, FSSchedulerNode node) {
try {
writeLock.lock();
Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(
schedulerKey);
- RMContainer reservedContainer = reservedContainers.remove(
- node.getNodeID());
- if (reservedContainers.isEmpty()) {
- this.reservedContainers.remove(schedulerKey);
- }
+ boolean unreserved = false;
+ if (reservedContainers != null) {
+ RMContainer reservedContainer = reservedContainers.remove(
+ node.getNodeID());
+ if (reservedContainers.isEmpty()) {
+ this.reservedContainers.remove(schedulerKey);
+ }
- // Reset the re-reservation count
- resetReReservations(schedulerKey);
+ // Reset the re-reservation count
+ resetReReservations(schedulerKey);
- Resource resource = reservedContainer.getContainer().getResource();
- this.attemptResourceUsage.decReserved(resource);
+ Resource resource = reservedContainer.getContainer().getResource();
+ this.attemptResourceUsage.decReserved(resource);
+ unreserved = true;
- LOG.info(
- "Application " + getApplicationId() + " unreserved " + " on node "
- + node + ", currently has " + reservedContainers.size()
- + " at priority " + schedulerKey.getPriority()
- + "; currentReservation " + this.attemptResourceUsage
- .getReserved());
+ LOG.info(
+ "Application " + getApplicationId() + " unreserved " + " on node "
+ + node + ", currently has " + reservedContainers.size()
+ + " at priority " + schedulerKey.getPriority()
+ + "; currentReservation " + this.attemptResourceUsage
+ .getReserved());
+ }
+ return unreserved;
} finally {
writeLock.unlock();
}
@@ -229,7 +235,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
SchedulingPolicy policy = fsQueue.getPolicy();
Resource queueFairShare = fsQueue.getFairShare();
- Resource queueUsage = fsQueue.getResourceUsage();
+ Resource queueUsage = fsQueue.getGuaranteedResourceUsage();
Resource clusterResource = this.scheduler.getClusterResource();
Resource clusterUsage = this.scheduler.getRootQueueMetrics()
.getAllocatedResources();
@@ -420,7 +426,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public RMContainer allocate(NodeType type, FSSchedulerNode node,
SchedulerRequestKey schedulerKey, PendingAsk pendingAsk,
- Container reservedContainer) {
+ Container reservedContainer, boolean opportunistic) {
RMContainer rmContainer;
Container container;
@@ -445,9 +451,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
container = reservedContainer;
+ ExecutionType executionType = opportunistic ?
+ ExecutionType.OPPORTUNISTIC : ExecutionType.GUARANTEED;
if (container == null) {
container = createContainer(node, pendingAsk.getPerAllocationResource(),
- schedulerKey);
+ schedulerKey, executionType);
}
// Create RMContainer
@@ -462,8 +470,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Update consumption and track allocations
ContainerRequest containerRequest = appSchedulingInfo.allocate(
type, node, schedulerKey, container);
- this.attemptResourceUsage.incUsed(container.getResource());
- getQueue().incUsedResource(container.getResource());
+ if (executionType.equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.incUsed(container.getResource());
+ getQueue().incUsedGuaranteedResource(container.getResource());
+ } else {
+ this.attemptOpportunisticResourceUsage.incUsed(container.getResource());
+ }
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl) rmContainer).setContainerRequest(containerRequest);
@@ -620,7 +632,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private Resource getUsageAfterPreemptingContainer(Resource containerResources,
Resource alreadyConsideringForPreemption) {
- Resource usageAfterPreemption = Resources.clone(getResourceUsage());
+ Resource usageAfterPreemption =
+ Resources.clone(getGuaranteedResourceUsage());
// Subtract resources of containers already queued for preemption
synchronized (preemptionVariablesLock) {
@@ -647,7 +660,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return Container
*/
private Container createContainer(FSSchedulerNode node, Resource capability,
- SchedulerRequestKey schedulerKey) {
+ SchedulerRequestKey schedulerKey, ExecutionType executionType) {
NodeId nodeId = node.getRMNode().getNodeID();
ContainerId containerId = BuilderUtils.newContainerId(
@@ -657,7 +670,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return BuilderUtils.newContainer(containerId, nodeId,
node.getRMNode().getHttpAddress(), capability,
schedulerKey.getPriority(), null,
- schedulerKey.getAllocationRequestId());
+ executionType, schedulerKey.getAllocationRequestId());
}
@Override
@@ -669,7 +682,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
super.recoverContainer(node, rmContainer);
if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
- getQueue().incUsedResource(rmContainer.getContainer().getResource());
+ getQueue().incUsedGuaranteedResource(
+ rmContainer.getContainer().getResource());
}
// If not running unmanaged, the first container we recover is always
@@ -707,7 +721,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (reservedContainer == null) {
reservedContainer =
createContainer(node, perAllocationResource,
- schedulerKey);
+ schedulerKey, ExecutionType.GUARANTEED);
getMetrics().reserveResource(node.getPartition(), getUser(),
reservedContainer.getResource());
RMContainer rmContainer =
@@ -764,11 +778,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public void unreserve(SchedulerRequestKey schedulerKey,
FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
- unreserveInternal(schedulerKey, node);
- node.unreserveResource(this);
- clearReservation(node);
- getMetrics().unreserveResource(node.getPartition(),
- getUser(), rmContainer.getContainer().getResource());
+ if (unreserveInternal(schedulerKey, node)) {
+ node.unreserveResource(this);
+ clearReservation(node);
+ getMetrics().unreserveResource(node.getPartition(),
+ getUser(), rmContainer.getContainer().getResource());
+ }
}
private void setReservation(SchedulerNode node) {
@@ -842,13 +857,15 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*/
private Resource assignContainer(
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
- boolean reserved, SchedulerRequestKey schedulerKey) {
+ boolean reserved, boolean opportunistic,
+ SchedulerRequestKey schedulerKey) {
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
// How much does the node have?
- Resource available = node.getUnallocatedResource();
+ Resource available = opportunistic ?
+ node.allowedResourceForOverAllocation() : node.getUnallocatedResource();
Container reservedContainer = null;
if (reserved) {
@@ -860,39 +877,46 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// Inform the application of the new container for this request
RMContainer allocatedContainer =
allocate(type, node, schedulerKey, pendingAsk,
- reservedContainer);
- if (allocatedContainer == null) {
- // Did the application need this resource?
- if (reserved) {
+ reservedContainer, opportunistic);
+
+ // delete the previous reservation, if any
+ if (reserved) {
+ unreserve(schedulerKey, node);
+ }
+
+ if (allocatedContainer != null) {
+ if (opportunistic) {
+ // if an OPPORTUNISTIC container is allocated, we need to
+ // unreserve anything that we may have reserved in our
+ // previous attempt to assign GUARANTEED containers for this
+ // scheduling request.
unreserve(schedulerKey, node);
}
+
+
+ // Inform the node
+ node.allocateContainer(allocatedContainer);
+
+ // If not running unmanaged, the first container we allocate
+ // is always the AM. Set amResource for this app and update
+ // the leaf queue's AM usage
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ setAMResource(capability);
+ getQueue().addAMResourceUsage(capability);
+ setAmRunning(true);
+ }
+
+ return capability;
+ } else {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format(
"Resource ask %s fits in available node resources %s, " +
- "but no container was allocated",
+ "but no container was allocated",
capability, available));
}
- return Resources.none();
- }
- // If we had previously made a reservation, delete it
- if (reserved) {
- unreserve(schedulerKey, node);
- }
-
- // Inform the node
- node.allocateContainer(allocatedContainer);
-
- // If not running unmanaged, the first container we allocate is always
- // the AM. Set the amResource for this app and update the leaf queue's AM
- // usage
- if (!isAmRunning() && !getUnmanagedAM()) {
- setAMResource(capability);
- getQueue().addAMResourceUsage(capability);
- setAmRunning(true);
+ return Resources.none();
}
-
- return capability;
}
if (LOG.isDebugEnabled()) {
@@ -903,7 +927,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// The desired container won't fit here, so reserve
// Reserve only, if app does not wait for preempted resources on the node,
// otherwise we may end up with duplicate reservations
- if (isReservable(capability) &&
+ if (isReservable(capability) && !opportunistic &&
!node.isPreemptedForApp(this) &&
reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
type, schedulerKey)) {
@@ -949,7 +973,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@SuppressWarnings("deprecation")
- private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
+ private Resource assignContainer(FSSchedulerNode node, boolean opportunistic,
+ boolean reserved) {
if (LOG.isTraceEnabled()) {
LOG.trace("Node offered to app: " + getName() + " reserved: " + reserved);
}
@@ -1012,7 +1037,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, nodeLocalPendingAsk, NodeType.NODE_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
if (!appSchedulingInfo.canDelayTo(schedulerKey, node.getRackName())) {
@@ -1029,7 +1054,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, rackLocalPendingAsk, NodeType.RACK_LOCAL,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
PendingAsk offswitchAsk = getPendingAsk(schedulerKey,
@@ -1049,7 +1074,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ ", app attempt id: " + this.attemptId);
}
return assignContainer(node, offswitchAsk, NodeType.OFF_SWITCH,
- reserved, schedulerKey);
+ reserved, opportunistic, schedulerKey);
}
}
@@ -1150,7 +1175,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getUnallocatedResource())) {
- assignContainer(node, true);
+ assignContainer(node, false, true);
}
return true;
}
@@ -1166,7 +1191,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource fairDemand = Resources.componentwiseMin(threshold, demand);
// Check if the queue is starved for fairshare
- boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand);
+ boolean starved =
+ isUsageBelowShare(getGuaranteedResourceUsage(), fairDemand);
if (!starved) {
lastTimeAtFairShare = now;
@@ -1178,8 +1204,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
fairshareStarvation = Resources.none();
} else {
// The app has been starved for longer than preemption-timeout.
- fairshareStarvation =
- Resources.subtractFromNonNegative(fairDemand, getResourceUsage());
+ fairshareStarvation = Resources.subtractFromNonNegative(fairDemand,
+ getGuaranteedResourceUsage());
}
return fairshareStarvation;
}
@@ -1198,7 +1224,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return true if the app is starved for fairshare, false otherwise
*/
boolean isStarvedForFairShare() {
- return isUsageBelowShare(getResourceUsage(), getFairShare());
+ return isUsageBelowShare(getGuaranteedResourceUsage(), getFairShare());
}
/**
@@ -1299,7 +1325,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* Get the current app's unsatisfied demand.
*/
Resource getPendingDemand() {
- return Resources.subtract(demand, getResourceUsage());
+ return Resources.subtract(demand, getGuaranteedResourceUsage());
}
@Override
@@ -1318,11 +1344,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return getCurrentConsumption();
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return attemptOpportunisticResourceUsage.getUsed();
+ }
+
+ @Override
public float getWeight() {
float weight = 1.0F;
@@ -1371,7 +1402,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
if (isOverAMShareLimit()) {
PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk();
updateAMDiagnosticMsg(amAsk.getPerAllocationResource(),
@@ -1383,7 +1414,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
return Resources.none();
}
- return assignContainer(node, false);
+ return assignContainer(node, opportunistic, false);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index cbc74d2..3834130 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -97,7 +97,7 @@ public class FSLeafQueue extends FSQueue {
// when an appAttempt is created for an application, we'd like to move
// it over from assignedApps to either runnableApps or nonRunnableApps
assignedApps.remove(app.getApplicationId());
- incUsedResource(app.getResourceUsage());
+ incUsedGuaranteedResource(app.getGuaranteedResourceUsage());
} finally {
writeLock.unlock();
}
@@ -132,7 +132,7 @@ public class FSLeafQueue extends FSQueue {
getMetrics().setAMResourceUsage(amResourceUsage);
}
- decUsedResource(app.getResourceUsage());
+ decUsedGuaranteedResource(app.getGuaranteedResourceUsage());
return runnable;
}
@@ -302,6 +302,42 @@ public class FSLeafQueue extends FSQueue {
return demand;
}
+ @Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(guaranteedResource, app.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSAppAttempt app : runnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ for (FSAppAttempt app : nonRunnableApps) {
+ Resources.addTo(opportunisticResource,
+ app.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
Resource getAmResourceUsage() {
return amResourceUsage;
}
@@ -335,14 +371,14 @@ public class FSLeafQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = none();
if (LOG.isDebugEnabled()) {
LOG.debug("Node " + node.getNodeName() + " offered to queue: " +
getName() + " fairShare: " + getFairShare());
}
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
@@ -350,7 +386,7 @@ public class FSLeafQueue extends FSQueue {
if (SchedulerAppUtils.isPlaceBlacklisted(sched, node, LOG)) {
continue;
}
- assigned = sched.assignContainer(node);
+ assigned = sched.assignContainer(node, opportunistic);
if (!assigned.equals(none())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigned container in queue:" + getName() + " " +
@@ -557,7 +593,7 @@ public class FSLeafQueue extends FSQueue {
Resource starvation =
Resources.componentwiseMin(getMinShare(), getDemand());
- Resources.subtractFromNonNegative(starvation, getResourceUsage());
+ Resources.subtractFromNonNegative(starvation, getGuaranteedResourceUsage());
boolean starved = !Resources.isNone(starvation);
long now = scheduler.getClock().getTime();
@@ -616,7 +652,7 @@ public class FSLeafQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", Runnable: " + getNumRunnableApps() +
", NumPendingApps: " + getNumPendingApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index d5df549..17a74c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -119,6 +119,34 @@ public class FSParentQueue extends FSQueue {
}
@Override
+ public Resource getGuaranteedResourceUsage() {
+ Resource guaranteedResource = Resources.createResource(0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(guaranteedResource, child.getGuaranteedResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return guaranteedResource;
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsage() {
+ Resource opportunisticResource = Resource.newInstance(0, 0);
+ readLock.lock();
+ try {
+ for (FSQueue child : childQueues) {
+ Resources.addTo(opportunisticResource,
+ child.getOpportunisticResourceUsage());
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return opportunisticResource;
+ }
+
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
@@ -177,11 +205,11 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
Resource assigned = Resources.none();
// If this queue is over its limit, reject
- if (!assignContainerPreCheck(node)) {
+ if (!assignContainerPreCheck(node, opportunistic)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assign container precheck on node " + node + " failed");
}
@@ -201,7 +229,7 @@ public class FSParentQueue extends FSQueue {
try {
sortedChildQueues.addAll(childQueues);
for (FSQueue child : sortedChildQueues) {
- assigned = child.assignContainer(node);
+ assigned = child.assignContainer(node, opportunistic);
if (!Resources.equals(assigned, Resources.none())) {
break;
}
@@ -285,7 +313,7 @@ public class FSParentQueue extends FSQueue {
", SteadyFairShare: " + getSteadyFairShare() +
", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
- ", ResourceUsage: " + getResourceUsage() +
+ ", Guaranteed ResourceUsage: " + getGuaranteedResourceUsage() +
", Demand: " + getDemand() +
", MaxAMShare: " + maxAMShare +
", Runnable: " + getNumRunnableApps() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 6b88a32..52c76f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -57,7 +57,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
private Resource steadyFairShare = Resources.createResource(0, 0);
private Resource reservedResource = Resources.createResource(0, 0);
- private final Resource resourceUsage = Resource.newInstance(0, 0);
+ private final Resource guaranteedResourceUsage = Resource.newInstance(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final YarnAuthorizationProvider authorizer;
@@ -235,7 +235,8 @@ public abstract class FSQueue implements Queue, Schedulable {
if (getFairShare().getMemorySize() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
- queueInfo.setCurrentCapacity((float) getResourceUsage().getMemorySize() /
+ queueInfo.setCurrentCapacity(
+ (float) getGuaranteedResourceUsage().getMemorySize() /
getFairShare().getMemorySize());
}
@@ -419,14 +420,17 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @return true if check passes (can assign) or false otherwise
*/
- boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (node.getReservedContainer() != null) {
+ boolean assignContainerPreCheck(FSSchedulerNode node, boolean opportunistic) {
+ if (opportunistic) {
+ // always pre-approve OPPORTUNISTIC containers to be assigned on the node
+ return true;
+ } else if (node.getReservedContainer() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because it has reserved containers.");
}
return false;
- } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) {
+ } else if (!Resources.fitsIn(getGuaranteedResourceUsage(), getMaxShare())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because queue resource usage is larger than MaxShare: "
@@ -449,7 +453,8 @@ public abstract class FSQueue implements Queue, Schedulable {
@Override
public String toString() {
return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]",
- getName(), getDemand(), getResourceUsage(), fairShare, getWeight());
+ getName(), getDemand(), getGuaranteedResourceUsage(), fairShare,
+ getWeight());
}
@Override
@@ -481,8 +486,8 @@ public abstract class FSQueue implements Queue, Schedulable {
}
@Override
- public Resource getResourceUsage() {
- return resourceUsage;
+ public Resource getGuaranteedResourceUsage() {
+ return guaranteedResourceUsage;
}
/**
@@ -490,11 +495,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to increase
*/
- protected void incUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.addTo(resourceUsage, res);
+ protected void incUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.addTo(guaranteedResourceUsage, res);
if (parent != null) {
- parent.incUsedResource(res);
+ parent.incUsedGuaranteedResource(res);
}
}
}
@@ -504,11 +509,11 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param res the resource to decrease
*/
- protected void decUsedResource(Resource res) {
- synchronized (resourceUsage) {
- Resources.subtractFrom(resourceUsage, res);
+ protected void decUsedGuaranteedResource(Resource res) {
+ synchronized (guaranteedResourceUsage) {
+ Resources.subtractFrom(guaranteedResourceUsage, res);
if (parent != null) {
- parent.decUsedResource(res);
+ parent.decUsedGuaranteedResource(res);
}
}
}
@@ -521,7 +526,7 @@ public abstract class FSQueue implements Queue, Schedulable {
boolean fitsInMaxShare(Resource additionalResource) {
Resource usagePlusAddition =
- Resources.add(getResourceUsage(), additionalResource);
+ Resources.add(getGuaranteedResourceUsage(), additionalResource);
if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) {
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 43a47ae..5d48fac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -173,7 +173,6 @@ public class FairScheduler extends
private float reservableNodesRatio; // percentage of available nodes
// an app can be reserved on
-
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
// Continuous Scheduling enabled or not
@Deprecated
@@ -196,6 +195,8 @@ public class FairScheduler extends
boolean maxAssignDynamic;
protected int maxAssign; // Max containers to assign per heartbeat
+ protected boolean oversubscriptionEnabled;
+
@VisibleForTesting
final MaxRunningAppsEnforcer maxRunningEnforcer;
@@ -1075,13 +1076,13 @@ public class FairScheduler extends
* resources for preempted containers.
* @param node Node to check
*/
- static void assignPreemptedContainers(FSSchedulerNode node) {
+ static void attemptToAssignPreemptedResources(FSSchedulerNode node) {
for (Entry<FSAppAttempt, Resource> entry :
node.getPreemptionList().entrySet()) {
FSAppAttempt app = entry.getKey();
Resource preemptionPending = Resources.clone(entry.getValue());
while (!app.isStopped() && !Resources.isNone(preemptionPending)) {
- Resource assigned = app.assignContainer(node);
+ Resource assigned = app.assignContainer(node, false);
if (Resources.isNone(assigned) ||
assigned.equals(FairScheduler.CONTAINER_RESERVED)) {
// Fail to assign, let's not try further
@@ -1113,49 +1114,85 @@ public class FairScheduler extends
// Assign new containers...
// 1. Ensure containers are assigned to the apps that preempted
// 2. Check for reserved applications
- // 3. Schedule if there are no reservations
+ // 3. Schedule GUARANTEED containers if there are no reservations
+ // 4. Schedule OPPORTUNISTIC containers if possible
// Apps may wait for preempted containers
// We have to satisfy these first to avoid cases, when we preempt
// a container for A from B and C gets the preempted containers,
// when C does not qualify for preemption itself.
- assignPreemptedContainers(node);
- FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
- boolean validReservation = false;
- if (reservedAppSchedulable != null) {
- validReservation = reservedAppSchedulable.assignReservedContainer(node);
- }
+ attemptToAssignPreemptedResources(node);
+
+ boolean validReservation = attemptToAssignReservedResources(node);
if (!validReservation) {
- // No reservation, schedule at queue which is farthest below fair share
- int assignedContainers = 0;
- Resource assignedResource = Resources.clone(Resources.none());
- Resource maxResourcesToAssign = Resources.multiply(
- node.getUnallocatedResource(), 0.5f);
-
- while (node.getReservedContainer() == null) {
- Resource assignment = queueMgr.getRootQueue().assignContainer(node);
-
- if (assignment.equals(Resources.none())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No container is allocated on node " + node);
- }
- break;
- }
+ // only attempt to assign GUARANTEED containers if there is no
+ // reservation on the node because
+ attemptToAssignResourcesAsGuaranteedContainers(node);
+ }
- assignedContainers++;
- Resources.addTo(assignedResource, assignment);
- if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
- assignedResource)) {
- break;
- }
- }
+ // attempt to assign OPPORTUNISTIC containers regardless of whether
+ // we have made a reservation or assigned a GUARANTEED container
+ if (oversubscriptionEnabled) {
+ attemptToAssignResourcesAsOpportunisticContainers(node);
}
+
updateRootQueueMetrics();
} finally {
writeLock.unlock();
}
}
+ /**
+ * Assign the reserved resource to the application that have reserved it.
+ */
+ private boolean attemptToAssignReservedResources(FSSchedulerNode node) {
+ boolean success = false;
+ FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
+ if (reservedAppSchedulable != null) {
+ success = reservedAppSchedulable.assignReservedContainer(node);
+ }
+ return success;
+ }
+
+ private void attemptToAssignResourcesAsGuaranteedContainers(
+ FSSchedulerNode node) {
+ // No reservation, schedule at queue which is farthest below fair share
+ int assignedContainers = 0;
+ Resource assignedResource = Resources.clone(Resources.none());
+ Resource maxResourcesToAssign = Resources.multiply(
+ node.getUnallocatedResource(), 0.5f);
+ while (node.getReservedContainer() == null) {
+ Resource assignment =
+ queueMgr.getRootQueue().assignContainer(node, false);
+ if (assignment.equals(Resources.none())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No container is allocated on node " + node);
+ }
+ break;
+ }
+ assignedContainers++;
+ Resources.addTo(assignedResource, assignment);
+
+ if (!shouldContinueAssigning(assignedContainers, maxResourcesToAssign,
+ assignedResource)) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Try to assign OPPORTUNISTIC containers as long as there is resources
+ * to.
+ * @param node the node to assign OPPORTUNISTIC containers on
+ */
+ private void attemptToAssignResourcesAsOpportunisticContainers(
+ FSSchedulerNode node) {
+ while (!Resources.none().equals(
+ queueMgr.getRootQueue().assignContainer(node, true))) {
+ // nothing to do here
+ }
+ }
+
public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
return super.getApplicationAttempt(appAttemptId);
}
@@ -1398,6 +1435,7 @@ public class FairScheduler extends
sizeBasedWeight = this.conf.getSizeBasedWeight();
usePortForNodeName = this.conf.getUsePortForNodeName();
reservableNodesRatio = this.conf.getReservableNodes();
+ oversubscriptionEnabled = this.conf.isOversubscriptionEnabled();
updateInterval = this.conf.getUpdateInterval();
if (updateInterval < 0) {
@@ -1801,7 +1839,8 @@ public class FairScheduler extends
}
// maxShare
- if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption),
+ if (!Resources.fitsIn(
+ Resources.add(cur.getGuaranteedResourceUsage(), consumption),
cur.getMaxShare())) {
throw new YarnException("Moving app attempt " + appAttId + " to queue "
+ queueName + " would violate queue maxShare constraints on"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 8c4932b..48f91ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -348,6 +348,11 @@ public class FairSchedulerConfiguration extends Configuration {
DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS);
}
+ public boolean isOversubscriptionEnabled() {
+ return getBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED);
+ }
+
/**
* Delay in milliseconds for locality fallback node to rack.
* @deprecated linked to {@link #CONTINUOUS_SCHEDULING_ENABLED} deprecation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
index bd1ff7a..f018929 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
@@ -58,8 +58,17 @@ public interface Schedulable {
*/
Resource getDemand();
- /** Get the aggregate amount of resources consumed by the schedulable. */
- Resource getResourceUsage();
+ /**
+ * Get the aggregate amount of guaranteed resources consumed by the
+ * schedulable.
+ */
+ Resource getGuaranteedResourceUsage();
+
+ /**
+ * Get the aggregate amount of opportunistic resources consumed by the
+ * schedulable.
+ */
+ Resource getOpportunisticResourceUsage();
/** Minimum Resource share assigned to the schedulable. */
Resource getMinShare();
@@ -89,8 +98,10 @@ public interface Schedulable {
/**
* Assign a container on this node if possible, and return the amount of
* resources assigned.
+ * @param node the node to assign containers on
+ * @param opportunistic whether to assign OPPORTUNISTIC containers or not
*/
- Resource assignContainer(FSSchedulerNode node);
+ Resource assignContainer(FSSchedulerNode node, boolean opportunistic);
/** Get the fair share assigned to this Schedulable. */
Resource getFairShare();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 59635d9..f366891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -169,8 +169,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
extends DominantResourceFairnessComparator {
@Override
public int compare(Schedulable s1, Schedulable s2) {
- Resource usage1 = s1.getResourceUsage();
- Resource usage2 = s2.getResourceUsage();
+ Resource usage1 = s1.getGuaranteedResourceUsage();
+ Resource usage2 = s2.getGuaranteedResourceUsage();
Resource minShare1 = s1.getMinShare();
Resource minShare2 = s2.getMinShare();
Resource clusterCapacity = fsContext.getClusterResource();
@@ -370,9 +370,9 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public int compare(Schedulable s1, Schedulable s2) {
ResourceInformation[] resourceInfo1 =
- s1.getResourceUsage().getResources();
+ s1.getGuaranteedResourceUsage().getResources();
ResourceInformation[] resourceInfo2 =
- s2.getResourceUsage().getResources();
+ s2.getGuaranteedResourceUsage().getResources();
ResourceInformation[] minShareInfo1 = s1.getMinShare().getResources();
ResourceInformation[] minShareInfo2 = s2.getMinShare().getResources();
ResourceInformation[] clusterInfo =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 0b5c10b..bce9c35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -93,8 +93,8 @@ public class FairSharePolicy extends SchedulingPolicy {
Resource resourceUsage2 = null;
if (res == 0) {
- resourceUsage1 = s1.getResourceUsage();
- resourceUsage2 = s2.getResourceUsage();
+ resourceUsage1 = s1.getGuaranteedResourceUsage();
+ resourceUsage2 = s2.getGuaranteedResourceUsage();
res = compareMinShareUsage(s1, s2, resourceUsage1, resourceUsage2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 913513c..fe313a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -88,7 +88,7 @@ public class FairSchedulerQueueInfo {
amMaxResources = new ResourceInfo(Resource.newInstance(
queue.getMetrics().getMaxAMShareMB(),
queue.getMetrics().getMaxAMShareVCores()));
- usedResources = new ResourceInfo(queue.getResourceUsage());
+ usedResources = new ResourceInfo(queue.getGuaranteedResourceUsage());
demandResources = new ResourceInfo(queue.getDemand());
fractionMemUsed = (float)usedResources.getMemorySize() /
clusterResources.getMemorySize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index ce1d6b5..bff6125 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -105,7 +104,10 @@ public class MockNodes {
return rs;
}
- private static class MockRMNodeImpl implements RMNode {
+ /**
+ * A mock implementation of RMNode.
+ */
+ public static class MockRMNodeImpl implements RMNode {
private NodeId nodeId;
private String hostName;
private String nodeAddr;
@@ -120,12 +122,27 @@ public class MockNodes {
private ResourceUtilization containersUtilization;
private ResourceUtilization nodeUtilization;
private Resource physicalResource;
+ private OverAllocationInfo overAllocationInfo;
+ private List<UpdatedContainerInfo> containerUpdates =
+ Collections.EMPTY_LIST;
+
+ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
+ Resource perNode, String rackName, String healthReport,
+ long lastHealthReportTime, int cmdPort, String hostName,
+ NodeState state, Set<String> labels,
+ ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ this(nodeId, nodeAddr, httpAddress, perNode, rackName, healthReport,
+ lastHealthReportTime, cmdPort, hostName, state, labels,
+ containersUtilization, nodeUtilization, pPhysicalResource, null);
+ }
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
Set<String> labels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization, Resource pPhysicalResource) {
+ ResourceUtilization nodeUtilization, Resource pPhysicalResource,
+ OverAllocationInfo overAllocationInfo) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
@@ -140,6 +157,7 @@ public class MockNodes {
this.containersUtilization = containersUtilization;
this.nodeUtilization = nodeUtilization;
this.physicalResource = pPhysicalResource;
+ this.overAllocationInfo = overAllocationInfo;
}
@Override
@@ -228,7 +246,7 @@ public class MockNodes {
@Override
public List<UpdatedContainerInfo> pullContainerUpdates() {
- return new ArrayList<UpdatedContainerInfo>();
+ return containerUpdates;
}
@Override
@@ -266,7 +284,7 @@ public class MockNodes {
@Override
public OverAllocationInfo getOverAllocationInfo() {
- return null;
+ return this.overAllocationInfo;
}
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
@@ -311,6 +329,19 @@ public class MockNodes {
public Resource getPhysicalResource() {
return this.physicalResource;
}
+
+ public void updateResourceUtilization(ResourceUtilization utilization) {
+ this.nodeUtilization = utilization;
+ }
+
+ public void updateContainersAndNodeUtilization(
+ UpdatedContainerInfo updatedContainerInfo,
+ ResourceUtilization resourceUtilization) {
+ if (updatedContainerInfo != null) {
+ containerUpdates = Collections.singletonList(updatedContainerInfo);
+ }
+ this.nodeUtilization = resourceUtilization;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
@@ -334,6 +365,15 @@ public class MockNodes {
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Resource physicalResource) {
+ return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
+ labels, containersUtilization, nodeUtilization, physicalResource, null);
+ }
+
+ private static MockRMNodeImpl buildRMNode(int rack, final Resource perNode,
+ NodeState state, String httpAddr, int hostnum, String hostName, int port,
+ Set<String> labels, ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization, Resource physicalResource,
+ OverAllocationInfo overAllocationInfo) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
@@ -346,9 +386,9 @@ public class MockNodes {
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state, labels,
- containersUtilization, nodeUtilization, physicalResource);
+ containersUtilization, nodeUtilization,
+ physicalResource, overAllocationInfo);
}
-
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state) {
return buildRMNode(rack, perNode, state, "N/A");
@@ -377,4 +417,10 @@ public class MockNodes {
return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0", hostnum, hostName, port);
}
+ public static MockRMNodeImpl newNodeInfo(int rack, final Resource perNode,
+ OverAllocationInfo overAllocationInfo) {
+ return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0",
+ NODE_ID++, null, 123, null, ResourceUtilization.newInstance(0, 0, 0.0f),
+ ResourceUtilization.newInstance(0, 0, 0.0f), null, overAllocationInfo);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index df409f5..077c01a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -508,7 +508,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
FSParentQueue root = scheduler.getQueueManager().getRootQueue();
// ************ check cluster used Resources ********
assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
- assertEquals(usedResources,root.getResourceUsage());
+ assertEquals(usedResources, root.getGuaranteedResourceUsage());
// ************ check app headroom ****************
FSAppAttempt schedulerAttempt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
index b1fc2d0..c1d628b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
@@ -84,7 +84,7 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource assignContainer(FSSchedulerNode node) {
+ public Resource assignContainer(FSSchedulerNode node, boolean opportunistic) {
return null;
}
@@ -114,11 +114,16 @@ public class FakeSchedulable implements Schedulable {
}
@Override
- public Resource getResourceUsage() {
+ public Resource getGuaranteedResourceUsage() {
return usage;
}
@Override
+ public Resource getOpportunisticResourceUsage() {
+ return Resource.newInstance(0, 0);
+ }
+
+ @Override
public long getStartTime() {
return startTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index f581935..71db786 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -222,7 +222,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
scheduler.handle(nodeEvent);
scheduler.handle(updateEvent);
- assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ oldQueue.getGuaranteedResourceUsage());
scheduler.update();
assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
@@ -231,8 +232,10 @@ public class TestAppRunnability extends FairSchedulerTestBase {
assertSame(targetQueue, app.getQueue());
assertFalse(oldQueue.isRunnableApp(app));
assertTrue(targetQueue.isRunnableApp(app));
- assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
- assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(0, 0),
+ oldQueue.getGuaranteedResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1),
+ targetQueue.getGuaranteedResourceUsage());
assertEquals(0, oldQueue.getNumRunnableApps());
assertEquals(1, targetQueue.getNumRunnableApps());
assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index 51ffd23..9bfbb91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -223,7 +223,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
- Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
+ Mockito.when(mockQueue.getGuaranteedResourceUsage()).thenReturn(queueUsage);
Mockito.when(mockScheduler.getClusterResource()).thenReturn
(clusterResource);
Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
@@ -305,7 +305,7 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
getApplicationId()));
FSAppAttempt app = scheduler.getSchedulerApp(id11);
assertNotNull(app);
- Resource queueUsage = app.getQueue().getResourceUsage();
+ Resource queueUsage = app.getQueue().getGuaranteedResourceUsage();
assertEquals(0, queueUsage.getMemorySize());
assertEquals(0, queueUsage.getVirtualCores());
SchedulerNode n1 = scheduler.getSchedulerNode(node1.getNodeID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 4a738ca..efe4ad1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -88,7 +88,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
- Mockito.when(app.getResourceUsage()).thenReturn(Resources.none());
+ Mockito.when(app.getGuaranteedResourceUsage()).thenReturn(Resources.none());
schedulable.addApp(app, true);
schedulable.addApp(app, true);
@@ -176,7 +176,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
@Override
public void run() {
for (int i=0; i < 500; i++) {
- schedulable.getResourceUsage();
+ schedulable.getGuaranteedResourceUsage();
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86555164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 6726f17..f79ba4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -98,7 +98,7 @@ public class TestFSSchedulerNode {
ApplicationAttemptId appAttemptId =
mock(ApplicationAttemptId.class);
when(starvingApp.getApplicationAttemptId()).thenReturn(appAttemptId);
- when(starvingApp.assignContainer(schedulerNode)).thenAnswer(
+ when(starvingApp.assignContainer(schedulerNode, false)).thenAnswer(
new Answer<Resource>() {
@Override
public Resource answer(InvocationOnMock invocationOnMock)
@@ -142,7 +142,7 @@ public class TestFSSchedulerNode {
}
private void allocateContainers(FSSchedulerNode schedulerNode) {
- FairScheduler.assignPreemptedContainers(schedulerNode);
+ FairScheduler.attemptToAssignPreemptedResources(schedulerNode);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/24] hadoop git commit: YARN-7337. Expose per-node over-allocation
info in Node Report Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff73fb94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff73fb94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff73fb94
Branch: refs/heads/YARN-1011
Commit: ff73fb94b043552256d11e490cc006f54cafe0ae
Parents: c7c178c
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 08:39:03 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++-
.../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++-----
.../src/main/proto/yarn_protos.proto | 7 +-
.../applications/distributedshell/Client.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 48 +++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++---
.../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++----
.../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++----
.../server/resourcemanager/ClientRMService.java | 23 +++--
.../resourcemanager/DefaultAMSProcessor.java | 22 +++--
.../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++----
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/SchedulerNodeReport.java | 50 +++++++---
.../webapp/dao/FifoSchedulerInfo.java | 9 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 17 ++--
.../server/resourcemanager/NodeManager.java | 4 +-
.../capacity/TestApplicationPriority.java | 66 ++++++++-----
.../capacity/TestCapacityScheduler.java | 32 ++++---
.../TestCapacitySchedulerMultiNodes.java | 10 +-
.../TestNodeLabelContainerAllocation.java | 40 ++++----
.../scheduler/fifo/TestFifoScheduler.java | 50 ++++++----
.../webapp/TestRMWebServicesNodes.java | 13 +--
22 files changed, 498 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index efea709..76b5493 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -124,11 +124,17 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").asInt());
+ n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+ 0, n.get("NumContainers").asInt());
+ Assert.assertEquals(n.get("NodeId") +
+ ": Unexpected number of opportunistic containers used",
+ 0, n.get("NumOpportunisticContainers").asInt());
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected amount of used memory",
+ n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
0, n.get("UsedMemoryMB").asInt());
+ Assert.assertEquals(
+ n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+ 0, n.get("UsedOpportunisticMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
+ Assert.assertNull(n.get("NumOpportunisticContainers"));
+ Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 625ad23..dfd9607 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link NodeId} of the node.</li>
* <li>HTTP Tracking URL of the node.</li>
* <li>Rack name for the node.</li>
- * <li>Used {@link Resource} on the node.</li>
+ * <li>Used guaranteed {@link Resource} on the node.</li>
+ * <li>Used opportunistic {@link Resource} on the node.</li>
* <li>Total available {@link Resource} of the node.</li>
- * <li>Number of running containers on the node.</li>
+ * <li>Number of total running containers on the node.</li>
+ * <li>Number of running guaranteed containers on the node.</li>
+ * <li>Number of running opportunistic containers on the node.</li>
* </ul>
*
* @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime) {
+ return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+ capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+ null, null, null, null, 0);
}
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+ Resource opportunisticUsed, int numOpportunisticContainers) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setDecommissioningTimeout(decommissioningTimeout);
nodeReport.setNodeUpdateType(nodeUpdateType);
+ nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
return nodeReport;
}
@@ -129,17 +136,43 @@ public abstract class NodeReport {
public abstract void setRackName(String rackName);
/**
- * Get <em>used</em> <code>Resource</code> on the node.
- * @return <em>used</em> <code>Resource</code> on the node
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsed();
@Private
@Unstable
+ @Deprecated
public abstract void setUsed(Resource used);
-
+
+ /**
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+ /**
+ * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+ * @return <em>opportunistic</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
/**
* Get the <em>total</em> <code>Resource</code> on the node.
* @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
public abstract void setCapability(Resource capability);
/**
- * Get the <em>number of allocated containers</em> on the node.
- * @return <em>number of allocated containers</em> on the node
+ * Get the <em>number of guaranteed containers</em> allocated on the node.
+ * @return <em>number of guaranteed containers</em> allocated on the node
*/
@Private
@Unstable
- public abstract int getNumContainers();
+ public abstract int getNumGuaranteedContainers();
@Private
@Unstable
- public abstract void setNumContainers(int numContainers);
-
+ public abstract void setNumGuaranteedContainers(int numContainers);
- /**
+ /**
+ * Get the <em>number of opportunistic containers</em> allocated on the node.
+ * @return <em>number of opportunistic containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public abstract int getNumOpportunisticContainers();
+
+ @Private
+ @Unstable
+ public abstract void setNumOpportunisticContainers(int numContainers);
+
+ /**
+ * Get the <em>number of containers</em> allocated on the node.
+ * @return <em>number of containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public int getNumTotalContainers() {
+ return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+ }
+
+ /**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5fe2cc9..49bf895 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -344,9 +344,9 @@ message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
optional string rackName = 3;
- optional ResourceProto used = 4;
+ optional ResourceProto guaranteedResourceUsed = 4;
optional ResourceProto capability = 5;
- optional int32 numContainers = 6;
+ optional int32 numGuaranteedContainers = 6;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
@@ -356,6 +356,9 @@ message NodeReportProto {
optional uint32 decommissioning_timeout = 13;
optional NodeUpdateTypeProto node_update_type = 14;
repeated NodeAttributeProto node_attributes = 15;
+ optional int32 numOpportunisticContainers = 16;
+ optional ResourceProto opportunisticResourceUsed = 17;
+ optional int32 numTotalContainers = 18;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index e8b69fe..60a8a08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -647,7 +647,7 @@ public class Client {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
- + ", nodeNumContainers=" + node.getNumContainers());
+ + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
}
QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 44e9870..9429a18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
@Private
@@ -184,7 +185,7 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
- .getNumContainers());
+ .getNumGuaranteedContainers());
}
writer.flush();
}
@@ -209,13 +210,18 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
- nodeReport.getNumContainers());
+ nodeReport.getNumGuaranteedContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
- writer.print("\tAllocated Resources : ");
- if (nodeReport.getUsed() != null) {
- writer.print(nodeReport.getUsed());
+ writer.print("\tAllocated Guaranteed Resources : ");
+ if (nodeReport.getGuaranteedResourceUsed() != null) {
+ writer.print(nodeReport.getGuaranteedResourceUsed());
+ }
+ writer.println();
+ writer.print("\tAllocated Opportunistic Resources : ");
+ if (nodeReport.getOpportunisticResourceUsed() != null) {
+ writer.print(nodeReport.getOpportunisticResourceUsed());
}
writer.println();
@@ -286,16 +292,32 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getHealthReport());
- nodeReportStr.print("\tContainers : ");
- nodeReportStr.println(nodeReport.getNumContainers());
- nodeReportStr.print("\tMemory-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ nodeReportStr.print("\tGuaranteed Containers : ");
+ nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+ nodeReportStr.print("\tOpportunistic Containers : ");
+ nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+ Resource guaranteedResourceUsed =
+ nodeReport.getGuaranteedResourceUsed();
+ Resource opportunisticResourceUsed =
+ nodeReport.getOpportunisticResourceUsed();
+ nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0MB" :
+ (guaranteedResourceUsed.getMemorySize() + "MB"));
+ nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0MB" :
+ (opportunisticResourceUsed.getMemorySize() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
- nodeReportStr.print("\tCPU-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
- : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0 vcores" :
+ (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0 vcores" :
+ (opportunisticResourceUsed.getVirtualCores() + " vcores"));
nodeReportStr.print("\tCPU-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.print("\tNode-Labels : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index a600895..6a24837 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1355,7 +1355,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1363,7 +1364,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1564,10 +1566,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tNode Attributes : rm.yarn.io/GPU(STRING)=ARM");
@@ -1602,10 +1607,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
pw.println("\tNode Attributes : ");
@@ -1639,10 +1647,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tNode Attributes : ");
@@ -2078,10 +2089,12 @@ public class TestYarnCLI {
// ordered
nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
}
- NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
- "rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+ "rack1", Records.newRecord(Resource.class),
+ Records.newRecord(Resource.class), 0, "",
+ 0, nodeLabels, null, null,
+ Records.newRecord(Resource.class), 0);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index 7d5a06a..ec0560e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -49,7 +49,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeReportProto.Builder builder = null;
private boolean viaProto = false;
private NodeId nodeId;
- private Resource used;
+ private Resource guaranteedResourceUsed;
+ private Resource opportunisticResourceUsed;
private Resource capability;
private ResourceUtilization containersUtilization = null;
private ResourceUtilization nodeUtilization = null;
@@ -114,9 +115,17 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public int getNumContainers() {
+ public int getNumGuaranteedContainers() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+ return (p.hasNumGuaranteedContainers()) ?
+ p.getNumGuaranteedContainers() : 0;
+ }
+
+ @Override
+ public int getNumOpportunisticContainers() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNumOpportunisticContainers()) ?
+ p.getNumOpportunisticContainers() : 0;
}
@Override
@@ -125,18 +134,25 @@ public class NodeReportPBImpl extends NodeReport {
return (p.hasRackName()) ? p.getRackName() : null;
}
+ @Deprecated
@Override
public Resource getUsed() {
- if (this.used != null) {
- return this.used;
+ return getGuaranteedResourceUsed();
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsed() {
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
}
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUsed()) {
+ if (!p.hasOpportunisticResourceUsed()) {
return null;
}
- this.used = convertFromProtoFormat(p.getUsed());
- return this.used;
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getOpportunisticResourceUsed());
+ return this.opportunisticResourceUsed;
}
@Override
@@ -201,13 +217,24 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public void setNumContainers(int numContainers) {
+ public void setNumGuaranteedContainers(int numContainers) {
+ maybeInitBuilder();
+ if (numContainers == 0) {
+ builder.clearNumGuaranteedContainers();
+ return;
+ }
+ builder.setNumGuaranteedContainers(numContainers);
+ }
+
+ @Override
+ public void setNumOpportunisticContainers(int numContainers) {
maybeInitBuilder();
if (numContainers == 0) {
- builder.clearNumContainers();
+ builder.clearNumOpportunisticContainers();
return;
}
- builder.setNumContainers(numContainers);
+ builder.setNumOpportunisticContainers(numContainers);
+
}
@Override
@@ -220,13 +247,43 @@ public class NodeReportPBImpl extends NodeReport {
builder.setRackName(rackName);
}
+ @Deprecated
@Override
public void setUsed(Resource used) {
+ setGuaranteedResourceUsed(used);
+ }
+
+ @Override
+ public Resource getGuaranteedResourceUsed() {
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasGuaranteedResourceUsed()) {
+ return null;
+ }
+ this.guaranteedResourceUsed =
+ convertFromProtoFormat(p.getGuaranteedResourceUsed());
+ return this.guaranteedResourceUsed;
+ }
+
+ @Override
+ public void setGuaranteedResourceUsed(Resource guaranteed) {
+ maybeInitBuilder();
+ if (guaranteedResourceUsed == null) {
+ builder.clearGuaranteedResourceUsed();
+ }
+ this.guaranteedResourceUsed = guaranteed;
+ }
+
+ @Override
+ public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
maybeInitBuilder();
- if (used == null) {
- builder.clearUsed();
+ if (opportunisticUsed == null) {
+ builder.clearOpportunisticResourceUsed();
}
- this.used = used;
+ this.opportunisticResourceUsed = opportunisticUsed;
}
public NodeReportProto getProto() {
@@ -263,8 +320,13 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null) {
- builder.setUsed(convertToProtoFormat(this.used));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setGuaranteedResourceUsed(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setOpportunisticResourceUsed(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index b9f35a5..ff88225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -186,26 +186,23 @@ public class BuilderUtils {
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainer,
+ String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+ guaranteedResourceUsed, capability, numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+ lastHealthReportTime, nodeLabels, null, null,
+ decommissioningTimeout, nodeUpdateType, null);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null, decommissioningTimeout, nodeUpdateType, null);
- }
-
- public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainers,
+ String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
NodeUpdateType nodeUpdateType, Set<NodeAttribute> attrs) {
@@ -214,9 +211,11 @@ public class BuilderUtils {
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
+ nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+ nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 4075332..1d3fef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1045,18 +1045,27 @@ public class ClientRMService extends AbstractService implements
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
- }
+ guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
+ }
Set<NodeAttribute> attrs = rmNode.getAllNodeAttributes();
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed,
+ rmNode.getTotalCapability(), numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index b2c5ef3..ddfabdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -365,17 +365,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ guaranteedResourceUsed =
+ schedulerNodeReport.getGuaranteedResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeId nodeId = rmNode.getNodeID();
NodeReport report =
BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed, rmNode.getTotalCapability(),
+ numGuaranteedContainers, opportunisticResourceUsed,
+ numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
rmNodeEntry.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..849db5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
for (final RMNode ni : nodes) {
- SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
- InfoMap info = new InfoMap();
- info.put("HostName", ni.getHostName());
- info.put("Rack", ni.getRackName());
- info.put("State", ni.getState().toString());
- info.put("NodeId", ni.getNodeID());
- info.put("NodeHTTPAddress", ni.getHttpAddress());
- info.put("LastHealthUpdate",
- ni.getLastHealthReportTime());
- info.put("HealthReport",
- ni.getHealthReport());
- info.put("NodeManagerVersion",
- ni.getNodeManagerVersion());
- if(report != null) {
- info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
- info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemorySize());
- }
+ SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+ InfoMap info = new InfoMap();
+ info.put("HostName", ni.getHostName());
+ info.put("Rack", ni.getRackName());
+ info.put("State", ni.getState().toString());
+ info.put("NodeId", ni.getNodeID());
+ info.put("NodeHTTPAddress", ni.getHttpAddress());
+ info.put("LastHealthUpdate",
+ ni.getLastHealthReportTime());
+ info.put("HealthReport",
+ ni.getHealthReport());
+ info.put("NodeManagerVersion",
+ ni.getNodeManagerVersion());
+ if(report != null) {
+ info.put("NumContainers", report.getNumGuaranteedContainers());
+ info.put("NumOpportunisticContainers",
+ report.getNumOpportunisticContainers());
+ info.put("UsedMemoryMB",
+ report.getGuaranteedResourceUsed().getMemorySize());
+ info.put("UsedOpportunisticMemoryMB",
+ report.getOpportunisticResourceUsed().getMemorySize());
+ info.put("AvailableMemoryMB",
+ report.getAvailableGuaranteedResource().getMemorySize());
+ }
- nodesInfo.add(info);
+ nodesInfo.add(info);
}
return JSON.toString(nodesInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b53b32b..0e32f1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -142,7 +142,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/** Physical resources in the node. */
private volatile Resource physicalResource;
- /* Container Queue Information for the node.. Used by Distributed Scheduler */
+ /* Container Queue Information for the node..*/
private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index ea30d78..25621a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Stable
public class SchedulerNodeReport {
- private final Resource used;
- private final Resource avail;
- private final int num;
+ private final Resource guaranteedResourceUsage;
+ private final Resource opportunisticResourceUsage;
+ private final Resource guaranteedResourceAvail;
+ private final int numOfGuaranteedContainers;
+ private final int numOfOpportunisticContainers;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getAllocatedResource();
- this.avail = node.getUnallocatedResource();
- this.num = node.getNumGuaranteedContainers();
+ this.guaranteedResourceUsage = node.getAllocatedResource();
+ this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+ this.guaranteedResourceAvail = node.getUnallocatedResource();
+ this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+ this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
}
/**
- * @return the amount of resources currently used by the node.
+ * @return the amount of guaranteed resources currently used by the node.
*/
- public Resource getUsedResource() {
- return used;
+ public Resource getGuaranteedResourceUsed() {
+ return guaranteedResourceUsage;
}
/**
- * @return the amount of resources currently available on the node
+ * @return the amount of opportunistic resources currently used by the node.
*/
- public Resource getAvailableResource() {
- return avail;
+ public Resource getOpportunisticResourceUsed() {
+ return opportunisticResourceUsage;
}
/**
- * @return the number of containers currently running on this node.
+ * @return the amount of guaranteed resources currently available on the node
*/
- public int getNumContainers() {
- return num;
+ public Resource getAvailableGuaranteedResource() {
+ return guaranteedResourceAvail;
+ }
+
+ /**
+ * @return the number of guaranteed containers currently running on
+ * this node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numOfGuaranteedContainers;
+ }
+
+ /**
+ * @return the number of opportunistic containers currently running on
+ * this node.
+ */
+ public int getNumOpportunisticContainers() {
+ return numOfOpportunisticContainers;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
index 1752546..f5d8d07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
for (RMNode ni : rmContext.getRMNodes().values()) {
SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
- this.usedNodeCapacity += report.getUsedResource().getMemorySize();
- this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+ this.usedNodeCapacity +=
+ report.getGuaranteedResourceUsed().getMemorySize();
+ this.availNodeCapacity +=
+ report.getAvailableGuaranteedResource().getMemorySize();
this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
- this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ this.numContainers +=
+ fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 7e5d20a..5618a98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -75,14 +75,17 @@ public class NodeInfo {
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null) {
- this.numContainers = report.getNumContainers();
- this.usedMemoryMB = report.getUsedResource().getMemorySize();
- this.availMemoryMB = report.getAvailableResource().getMemorySize();
- this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.numContainers = report.getNumGuaranteedContainers();
+ this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+ this.availMemoryMB =
+ report.getAvailableGuaranteedResource().getMemorySize();
+ this.usedVirtualCores =
+ report.getGuaranteedResourceUsed().getVirtualCores();
this.availableVirtualCores =
- report.getAvailableResource().getVirtualCores();
- this.usedResource = new ResourceInfo(report.getUsedResource());
- this.availableResource = new ResourceInfo(report.getAvailableResource());
+ report.getAvailableGuaranteedResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+ this.availableResource =
+ new ResourceInfo(report.getAvailableGuaranteedResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index ee974e3..96613fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getAvailableResource().getMemorySize());
+ this.nodeId).getAvailableGuaranteedResource().getMemorySize());
Assert.assertEquals(used.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getUsedResource().getMemorySize());
+ this.nodeId).getGuaranteedResourceUsed().getMemorySize());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index cad0151..9c317e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -176,8 +176,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// send updated request for App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
// check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(8 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 7
Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
// check node report, 1 GB used and 7 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
Assert.assertEquals(2, allocated2.size());
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// kill 1 more
counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
// check node report, 14 GB used and 2 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(14 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 9e15856..def0419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1246,8 +1246,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1268,8 +1270,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1288,7 +1292,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+ if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1297,8 +1301,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
}
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1314,9 +1320,11 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2816,8 +2824,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
index c90af94..f5b73ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
@@ -123,9 +123,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(2 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(8 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Ideally thread will invoke this, but thread operates every 1sec.
// Hence forcefully recompute nodes.
@@ -137,9 +138,10 @@ public class TestCapacitySchedulerMultiNodes extends CapacitySchedulerTestBase {
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
// Ideally thread will invoke this, but thread operates every 1sec.
// Hence forcefully recompute nodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 02c0cc5..8023457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(5 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(3 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(7 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(14 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Try to launch app2 in a2, asked 2GB, should success
// app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a2
cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
cs.getApplicationAttempt(am3.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a1
cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 3f8a552..ad68e31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -701,7 +701,8 @@ public class TestFifoScheduler {
am1.registerAppAttempt();
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -711,7 +712,8 @@ public class TestFifoScheduler {
am2.registerAppAttempt();
SchedulerNodeReport report_nm2 =
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -747,11 +749,15 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getAvailableGuaranteedResource().getMemorySize());
- Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -769,7 +775,8 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -826,7 +833,8 @@ public class TestFifoScheduler {
int checkAlloc =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -1117,8 +1125,10 @@ public class TestFifoScheduler {
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1139,8 +1149,10 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1159,7 +1171,7 @@ public class TestFifoScheduler {
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
if (null != report_nm1 &&
- report_nm1.getAvailableResource().getMemorySize() != 0) {
+ report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1169,8 +1181,10 @@ public class TestFifoScheduler {
// Now, the used resource is still 4 GB, and available resource is minus
// value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource
// over-commitment.
@@ -1188,9 +1202,11 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff73fb94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 72b7a45..057c5ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -817,15 +817,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
if (report != null) {
assertEquals("numContainers doesn't match: " + numContainers,
- report.getNumContainers(), numContainers);
+ report.getNumGuaranteedContainers(), numContainers);
assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
- .getUsedResource().getMemorySize(), usedMemoryMB);
+ .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
- .getAvailableResource().getMemorySize(), availMemoryMB);
+ .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
- .getUsedResource().getVirtualCores(), usedVirtualCores);
- assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
- .getAvailableResource().getVirtualCores(), availVirtualCores);
+ .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+ report.getAvailableGuaranteedResource().getVirtualCores(),
+ availVirtualCores);
}
if (opportunisticStatus != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/24] hadoop git commit: YARN-8240. Add queue-level control to
allow all applications in a queue to opt-out. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-8240. Add queue-level control to allow all applications in a queue to opt-out. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddae86c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddae86c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddae86c9
Branch: refs/heads/YARN-1011
Commit: ddae86c9bfb99d23b54bba6fda0c5a8230fee63e
Parents: f885133
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Jun 4 11:20:32 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:14:59 2018 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 9 ++
.../scheduler/fair/FSLeafQueue.java | 5 +
.../resourcemanager/scheduler/fair/FSQueue.java | 21 +++-
.../allocation/AllocationFileQueueParser.java | 4 +
.../fair/allocation/QueueProperties.java | 13 +++
.../webapp/dao/FairSchedulerQueueInfo.java | 6 ++
.../fair/TestAllocationFileLoaderService.java | 61 +++++++++++
.../scheduler/fair/TestFairScheduler.java | 103 +++++++++++++++++++
.../scheduler/fair/TestQueueManager.java | 70 +++++++++++++
.../src/site/markdown/FairScheduler.md | 1 +
10 files changed, 292 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index e48e04b..14e2aeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -103,6 +104,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private ReservationQueueConfiguration globalReservationQueueConfig;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
public AllocationConfiguration(QueueProperties queueProperties,
AllocationFileParser allocationFileParser,
@@ -138,6 +140,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
this.placementPolicy = newPlacementPolicy;
this.configuredQueues = queueProperties.getConfiguredQueues();
this.nonPreemptableQueues = queueProperties.getNonPreemptableQueues();
+ this.queueOversubscriptionSetting =
+ queueProperties.getQueueOversubscriptionSetting();
}
public AllocationConfiguration(Configuration conf) {
@@ -167,6 +171,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
placementPolicy =
QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
nonPreemptableQueues = new HashSet<>();
+ queueOversubscriptionSetting = new HashMap<>(0);
}
/**
@@ -221,6 +226,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return !nonPreemptableQueues.contains(queueName);
}
+ public Optional<Boolean> isOversubscriptionAllowed(String queueName) {
+ return Optional.ofNullable(queueOversubscriptionSetting.get(queueName));
+ }
+
private float getQueueWeight(String queue) {
Float weight = queueWeights.get(queue);
return (weight == null) ? 1.0f : weight;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 3834130..48836b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -378,6 +378,11 @@ public class FSLeafQueue extends FSQueue {
getName() + " fairShare: " + getFairShare());
}
+ if (opportunistic && !isOversubscriptionAllowed()) {
+ // do not proceed if the queue is opted out of over-subscription
+ return assigned;
+ }
+
if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 52c76f4..3a9ffae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -75,6 +76,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private ConfigurableResource maxShare;
protected int maxRunningApps;
private ConfigurableResource maxChildQueueResource;
+ private boolean isOversubscriptionAllowed;
// maxAMShare is a value between 0 and 1.
protected float maxAMShare;
@@ -93,6 +95,7 @@ public abstract class FSQueue implements Queue, Schedulable {
this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name);
this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
this.parent = parent;
+ this.isOversubscriptionAllowed = true;
setPolicy(scheduler.getAllocationConfiguration().getSchedulingPolicy(name));
reinit(false);
}
@@ -112,6 +115,14 @@ public abstract class FSQueue implements Queue, Schedulable {
allocConf.initFSQueue(this);
updatePreemptionVariables();
+ Optional<Boolean> allowed = allocConf.isOversubscriptionAllowed(name);
+ if (allowed.isPresent()) {
+ isOversubscriptionAllowed = allowed.get();
+ } else if (getParent() != null){
+ // inherit from the parent queue if it is not explicitly set
+ isOversubscriptionAllowed = getParent().isOversubscriptionAllowed();
+ }
+
if (recursive) {
for (FSQueue child : getChildQueues()) {
child.reinit(recursive);
@@ -213,6 +224,15 @@ public abstract class FSQueue implements Queue, Schedulable {
return 0;
}
+ public boolean isOversubscriptionAllowed() {
+ return isOversubscriptionAllowed;
+ }
+
+ @VisibleForTesting
+ public void allowOversubscription(boolean allowed) {
+ this.isOversubscriptionAllowed = allowed;
+ }
+
@Override
public Priority getPriority() {
Priority p = recordFactory.newRecordInstance(Priority.class);
@@ -584,7 +604,6 @@ public abstract class FSQueue implements Queue, Schedulable {
return sb.toString();
}
-
/**
* Recursively dump states of all queues.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index 441c34a..62a6060 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -68,6 +68,7 @@ public class AllocationFileQueueParser {
private static final String ACL_SUBMIT_RESERVATIONS = "aclSubmitReservations";
private static final String RESERVATION = "reservation";
private static final String ALLOW_PREEMPTION_FROM = "allowPreemptionFrom";
+ private static final String ALLOW_OVERSUBSCRIPTION ="allowOversubscription";
private static final String QUEUE = "queue";
private static final String POOL = "pool";
@@ -207,6 +208,9 @@ public class AllocationFileQueueParser {
if (!Boolean.parseBoolean(text)) {
builder.nonPreemptableQueues(queueName);
}
+ } else if (ALLOW_OVERSUBSCRIPTION.equals(field.getTagName())) {
+ Boolean allow = Boolean.parseBoolean(getTrimmedTextData(field));
+ builder.addQueueOversubscriptionSetting(queueName, allow);
} else if (QUEUE.endsWith(field.getTagName())
|| POOL.equals(field.getTagName())) {
loadQueue(queueName, field, builder);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
index ee5f179..f15f912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
@@ -52,6 +52,7 @@ public class QueueProperties {
reservationAcls;
private final Set<String> reservableQueues;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
private final Map<FSQueueType, Set<String>> configuredQueues;
QueueProperties(Builder builder) {
@@ -70,6 +71,7 @@ public class QueueProperties {
this.maxChildQueueResources = builder.maxChildQueueResources;
this.reservationAcls = builder.reservationAcls;
this.queueAcls = builder.queueAcls;
+ this.queueOversubscriptionSetting = builder.queueOversubscriptionSetting;
}
public Map<FSQueueType, Set<String>> getConfiguredQueues() {
@@ -133,6 +135,9 @@ public class QueueProperties {
return nonPreemptableQueues;
}
+ public Map<String, Boolean> getQueueOversubscriptionSetting() {
+ return queueOversubscriptionSetting;
+ }
/**
* Builder class for {@link QueueProperties}.
* All methods are adding queue properties to the maps of this builder
@@ -160,6 +165,8 @@ public class QueueProperties {
reservationAcls = new HashMap<>();
private Set<String> reservableQueues = new HashSet<>();
private Set<String> nonPreemptableQueues = new HashSet<>();
+ private Map<String, Boolean> queueOversubscriptionSetting =
+ new HashMap<>(0);
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
@@ -253,6 +260,12 @@ public class QueueProperties {
return this;
}
+ public Builder addQueueOversubscriptionSetting(
+ String queue, Boolean allow) {
+ this.queueOversubscriptionSetting.put(queue, allow);
+ return this;
+ }
+
public void configuredQueues(FSQueueType queueType, String queueName) {
this.configuredQueues.get(queueType).add(queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index fe313a6..45b9ea9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -68,6 +68,7 @@ public class FairSchedulerQueueInfo {
private String schedulingPolicy;
private boolean preemptable;
+ private boolean allowOversubscription;
private FairSchedulerQueueInfoList childQueues;
@@ -118,6 +119,7 @@ public class FairSchedulerQueueInfo {
}
preemptable = queue.isPreemptable();
+ allowOversubscription = queue.isOversubscriptionAllowed();
childQueues = getChildQueues(queue, scheduler);
}
@@ -255,4 +257,8 @@ public class FairSchedulerQueueInfo {
public boolean isPreemptable() {
return preemptable;
}
+
+ public boolean isOversubscriptionAllowed() {
+ return allowOversubscription;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 50a003e..35144cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -538,6 +538,67 @@ public class TestAllocationFileLoaderService {
}
@Test
+ public void testQueueOptOutOfOversubscription() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ // Queue A has the default value of allowOversubscription
+ out.println("<queue name=\"queueA\">");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ // The child queue of A, queueA.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\" />");
+ out.println("</queue>");
+ // Queue A has opted out of oversubscription
+ out.println("<queue name=\"queueB\">");
+ // The child queue of B, queueB.child has overridden the parent's setting
+ out.println("<allowOversubscription>true</allowOversubscription>");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\" />");
+ // The child queue of B, queueB.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ allocLoader.init(conf);
+ ReloadListener confHolder = new ReloadListener();
+ allocLoader.setReloadListener(confHolder);
+ allocLoader.reloadAllocations();
+ AllocationConfiguration queueConf = confHolder.allocConf;
+
+ assertTrue("allowOversubscription is not set for queueA",
+ !queueConf.isOversubscriptionAllowed("root.queueA").isPresent());
+ assertTrue("allowOversubscription is not set for queueA.child2",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child2").isPresent());
+ assertTrue("allowOversubscription is not set for queueB.child1",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child1").isPresent());
+
+ assertTrue("allowOversubscription is set for queueB",
+ queueConf.isOversubscriptionAllowed("root.queueB").isPresent());
+ assertTrue("queueB does allow oversubscription",
+ queueConf.isOversubscriptionAllowed("root.queueB").get());
+ assertTrue("allowOversubscription is set for queueB.child2",
+ queueConf.isOversubscriptionAllowed("root.queueB.child2").isPresent());
+ assertTrue("queueB.child2 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child2").get());
+
+ assertTrue("allowOversubscription is set for queueA.child1",
+ queueConf.isOversubscriptionAllowed("root.queueA.child1").isPresent());
+ assertTrue("queueA.child1 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child1").get());
+
+ }
+
+ @Test
public void testSimplePlacementPolicyFromConf() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9302f88..5878ccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2789,6 +2789,109 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ @Test
+ public void testQueueOptOutOfOversubscription() throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue queue1 =
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+ // queue 2 has opted out of oversubscription
+ FSLeafQueue queue2 =
+ scheduler.getQueueManager().getLeafQueue("queue2", true);
+ queue2.allowOversubscription(false);
+
+ FSParentQueue queue3
+ = scheduler.getQueueManager().getParentQueue("queue3", true);
+ // queue 3 has opted out of oversubscription
+ queue3.allowOversubscription(false);
+ FSLeafQueue queue4 =
+ scheduler.getQueueManager().getLeafQueue("queue3.queue4", true);
+ assertTrue("queue 4 should inherit the setting from queue 3",
+ !queue4.isOversubscriptionAllowed());
+
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request in queue1 that leaves some unallocated
+ // resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request in queue 2 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+
+ // create another scheduling request in queue 4 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1536, "queue3.queue4", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3.queue4").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 3674ffb..10111f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -642,4 +642,74 @@ public class TestQueueManager {
assertNotNull("root.a does not exist", p);
}
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is enabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsEnabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should allow oversubscription as its parent",
+ queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue("root.queue1.queue2 should allow oversubscription as its parent",
+ queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
+
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is disabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsDisabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+ q1.allowOversubscription(false);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should not allow oversubscription",
+ !queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue(
+ "root.queue1.queue2 should not allow oversubscription as its parent",
+ !queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddae86c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index b5bcbf5..920b2ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -111,6 +111,7 @@ The allocation file must be in XML format. The format contains five types of ele
* **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is 0.5f.
* **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues.
+ * **allowOversubscription**: determines whether the scheduler is allowed to assign OPPORTUNISTIC resources to applications running in the queue when over-subscription is turned on. The default is true.
* **reservation**: indicates to the `ReservationSystem` that the queue's resources is available for users to reserve. This only applies for leaf queues. A leaf queue is not reservable if this property isn't configured.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/24] hadoop git commit: YARN-7460. Exclude findbugs warnings on
SchedulerNode.numGuaranteedContainers and numOpportunisticContainers.
Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7460. Exclude findbugs warnings on SchedulerNode.numGuaranteedContainers and numOpportunisticContainers. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7c178c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7c178c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7c178c1
Branch: refs/heads/YARN-1011
Commit: c7c178c14e1455e7ffb27fc932725123a316d625
Parents: 8655516
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Nov 20 13:27:32 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:11 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c178c1/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 216c3bd..ce3a3b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -71,6 +71,16 @@
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numOpportunisticContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numGuaranteedContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl$AppRejectedTransition" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/24] hadoop git commit: YARN-6672. Add NM preemption of
opportunistic containers when utilization goes high.
Posted by ha...@apache.org.
YARN-6672. Add NM preemption of opportunistic containers when utilization goes high.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a54014d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a54014d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a54014d
Branch: refs/heads/YARN-1011
Commit: 0a54014dd54234f63e2d473781a80b6c7fc5c9c9
Parents: 98218c2
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Jul 2 15:37:47 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:02 2018 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 85 ++-
.../AllocationBasedResourceTracker.java | 5 +
.../scheduler/ContainerScheduler.java | 59 +-
.../scheduler/ContainerSchedulerEventType.java | 4 +-
...rSchedulerOverallocationPreemptionEvent.java | 45 ++
.../scheduler/NMAllocationPreemptionPolicy.java | 54 ++
.../scheduler/ResourceUtilizationTracker.java | 7 +
...shotBasedOverAllocationPreemptionPolicy.java | 81 +++
.../BaseContainerManagerTest.java | 126 ++++
.../TestContainerSchedulerQueuing.java | 167 +++--
...estContainerSchedulerWithOverAllocation.java | 650 +++++++++++--------
...shotBasedOverAllocationPreemptionPolicy.java | 259 ++++++++
12 files changed, 1229 insertions(+), 313 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 8f72b56..c36dfd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -20,15 +20,20 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerOverallocationPreemptionEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPreemptionPolicy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPreemptionPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,6 +126,7 @@ public class ContainersMonitorImpl extends AbstractService implements
private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
+ private NMAllocationPreemptionPolicy overAllocationPreemptionPolicy;
private volatile boolean stopped = false;
@@ -375,6 +381,9 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPolicy =
createOverAllocationPolicy(resourceThresholds);
+ this.overAllocationPreemptionPolicy = createOverAllocationPreemptionPolicy(
+ overAllocationPreemptionThresholds, overAlloctionPreemptionCpuCount);
+
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -387,6 +396,12 @@ public class ContainersMonitorImpl extends AbstractService implements
return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
}
+ private NMAllocationPreemptionPolicy createOverAllocationPreemptionPolicy(
+ ResourceThresholds resourceThresholds, int maxTimesCpuOverLimit) {
+ return new SnapshotBasedOverAllocationPreemptionPolicy(
+ resourceThresholds, maxTimesCpuOverLimit, this);
+ }
+
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -673,9 +688,7 @@ public class ContainersMonitorImpl extends AbstractService implements
setLatestContainersUtilization(trackedContainersUtilization);
// check opportunity to start containers if over-allocation is on
- if (context.isOverAllocationEnabled()) {
- attemptToStartContainersUponLowUtilization();
- }
+ checkUtilization();
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1076,13 +1089,46 @@ public class ContainersMonitorImpl extends AbstractService implements
return overAllocationPolicy;
}
+ public NMAllocationPreemptionPolicy getOverAllocationPreemptionPolicy() {
+ return overAllocationPreemptionPolicy;
+ }
+
private void setLatestContainersUtilization(ResourceUtilization utilization) {
this.latestContainersUtilization = new ContainersResourceUtilization(
- utilization, System.currentTimeMillis());
+ utilization, Time.now());
}
+ /**
+ * Check the resource utilization of the node. If the utilization is below
+ * the over-allocation threshold, {@link ContainerScheduler} is notified to
+ * launch OPPORTUNISTIC containers that are being queued to bring the
+ * utilization up to the over-allocation threshold. If the utilization
+ * is above the preemption threshold, {@link ContainerScheduler} is notified
+ * to preempt running OPPORTUNISTIC containers in order to bring the node
+ * utilization down to the preemption threshold.
+ * @return true if the utilization is below the over-allocation threshold or
+ * above the preemption threshold
+ * false otherwise
+ */
@VisibleForTesting
- public void attemptToStartContainersUponLowUtilization() {
+ public boolean checkUtilization() {
+ if (context.isOverAllocationEnabled()) {
+ return checkLowUtilization() || checkHighUtilization();
+ }
+ return false;
+ }
+
+ /**
+ * Check if the node resource utilization is below the over-allocation
+ * threshold. If so, a {@link ContainerSchedulerEvent} is
+ * generated so that OPPORTUNISTIC containers that are being queued can
+ * be launched by {@link ContainerScheduler} with over-allocation and
+ * the node utilization can be brought up to the over-allocation threshold
+ * @return true if the node utilization is below the over-allocation threshold
+ * false otherwise
+ */
+ private boolean checkLowUtilization() {
+ boolean opportunisticContainersToStart = false;
if (getContainerOverAllocationPolicy() != null) {
Resource available = getContainerOverAllocationPolicy()
.getAvailableResources();
@@ -1091,8 +1137,37 @@ public class ContainersMonitorImpl extends AbstractService implements
eventDispatcher.getEventHandler().handle(
new ContainerSchedulerEvent(null,
ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
+ opportunisticContainersToStart = true;
+ LOG.info("Node utilization is below its over-allocation threshold. " +
+ "Inform container scheduler to launch opportunistic containers.");
}
}
+ return opportunisticContainersToStart;
+ }
+
+ /**
+ * Check if the node resource utilization is over the preemption threshold.
+ * If so, a {@link ContainerSchedulerOverallocationPreemptionEvent} is
+ * generated so that OPPORTUNISTIC containers can be preempted by
+ * {@link ContainerScheduler} to reclaim resources in order to bring the
+ * node utilization down to the preemption threshold.
+ * @return true if the node utilization is over the preemption threshold
+ * false otherwise
+ */
+ private boolean checkHighUtilization() {
+ ResourceUtilization overLimit = getOverAllocationPreemptionPolicy()
+ .getResourcesToReclaim();
+
+ boolean opportunisticContainersToPreempt = false;
+
+ if (overLimit.getPhysicalMemory() > 0 || overLimit.getCPU() > 0) {
+ opportunisticContainersToPreempt = true;
+ eventDispatcher.getEventHandler().handle(
+ new ContainerSchedulerOverallocationPreemptionEvent(overLimit));
+ LOG.info("Node utilization is over the preemption threshold. " +
+ "Inform container scheduler to reclaim {}", overLimit);
+ }
+ return opportunisticContainersToPreempt;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
index a3e19eb..7840fef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -53,6 +53,11 @@ public class AllocationBasedResourceTracker
*/
@Override
public ResourceUtilization getCurrentUtilization() {
+ return getTotalAllocation();
+ }
+
+ @Override
+ public ResourceUtilization getTotalAllocation() {
return this.containersAllocation;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0d7de67..846efc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -222,12 +222,60 @@ public class ContainerScheduler extends AbstractService implements
// node utilization is low.
startOpportunisticContainers(utilizationTracker.getAvailableResources());
break;
+ case PREEMPT_CONTAINERS:
+ if (event instanceof ContainerSchedulerOverallocationPreemptionEvent) {
+ preemptOpportunisticContainers(
+ (ContainerSchedulerOverallocationPreemptionEvent) event);
+ } else {
+ LOG.error(
+ "Unknown event type on Preempt containers: {}", event.getType());
+ }
+ break;
default:
- LOG.error("Unknown event arrived at ContainerScheduler: "
- + event.toString());
+ LOG.error("Unknown event arrived at ContainerScheduler: {}", event);
}
}
+ private void preemptOpportunisticContainers(
+ ContainerSchedulerOverallocationPreemptionEvent event) {
+ ResourceUtilization resourcesToReclaim =
+ getResourcesToReclaim(event.getResourcesOverPreemptionThresholds());
+
+ List<Container> oppContainersToReclaim =
+ pickOpportunisticContainersToReclaimResources(
+ resourcesToReclaim);
+
+ killOpportunisticContainers(oppContainersToReclaim);
+ }
+
+ /**
+ * Get the amount of resources that need to be reclaimed by preempting
+ * OPPORTUNISTIC containers considering the amount of resources that
+ * are over the preemption thresholds and over the capacity of the node.
+ * When the node is not being over-allocated, its resource utilization
+ * can safely go to 100% without any OPPORTUNISTIC containers being killed.
+ */
+ private ResourceUtilization getResourcesToReclaim(
+ ResourceUtilization resourcesOverPreemptionThresholds) {
+ ResourceUtilization totalAllocation = ResourceUtilization.newInstance(
+ utilizationTracker.getTotalAllocation());
+ getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
+ totalAllocation);
+ ResourceUtilization overAllocatedResources =
+ ResourceUtilization.newInstance(
+ Math.max(0, totalAllocation.getPhysicalMemory()),
+ Math.max(0, totalAllocation.getVirtualMemory()),
+ Math.max(0, totalAllocation.getCPU()));
+
+ return ResourceUtilization.newInstance(
+ Math.min(overAllocatedResources.getPhysicalMemory(),
+ resourcesOverPreemptionThresholds.getPhysicalMemory()),
+ Math.min(overAllocatedResources.getVirtualMemory(),
+ resourcesOverPreemptionThresholds.getVirtualMemory()),
+ Math.min(overAllocatedResources.getCPU(),
+ resourcesOverPreemptionThresholds.getCPU()));
+ }
+
/**
* We assume that the ContainerManager has already figured out what kind
* of update this is.
@@ -601,8 +649,9 @@ public class ContainerScheduler extends AbstractService implements
@SuppressWarnings("unchecked")
private void reclaimOpportunisticContainerResources() {
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources();
+ pickOpportunisticContainersToReclaimResources(resourcesToFreeUp);
killOpportunisticContainers(extraOppContainersToReclaim);
}
@@ -642,12 +691,12 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources() {
+ private List<Container> pickOpportunisticContainersToReclaimResources(
+ ResourceUtilization resourcesToFreeUp) {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 9ad4f91..f76727d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -30,5 +30,7 @@ public enum ContainerSchedulerEventType {
CONTAINER_PAUSED,
RECOVERY_COMPLETED,
// Producer: Containers Monitor when over-allocation is on
- SCHEDULE_CONTAINERS
+ SCHEDULE_CONTAINERS,
+ // Producer: Containers Monitor when over-allocation is on
+ PREEMPT_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
new file mode 100644
index 0000000..547036a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerOverallocationPreemptionEvent.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+
+/**
+ * A {@link ContainerSchedulerEvent} generated by {@link ContainersMonitorImpl}
+ * when overallocation is turned on and the utilization goes over the
+ * proactive preemption thresholds.
+ */
+public class ContainerSchedulerOverallocationPreemptionEvent
+ extends ContainerSchedulerEvent {
+ private final ResourceUtilization resourcesOverPreemptionThresholds;
+ /**
+ * Create instance of Event.
+ *
+ * @param toFree resource to free up.
+ */
+ public ContainerSchedulerOverallocationPreemptionEvent(
+ ResourceUtilization toFree) {
+ super(null, ContainerSchedulerEventType.PREEMPT_CONTAINERS);
+ this.resourcesOverPreemptionThresholds = toFree;
+ }
+
+ public ResourceUtilization getResourcesOverPreemptionThresholds() {
+ return resourcesOverPreemptionThresholds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..61f6298
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPreemptionPolicy.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * Keeps track of containers utilization over time and determines how many
+ * resources need to be reclaimed by preempting opportunistic containers
+ * when over-allocation is turned on.
+ */
+public abstract class NMAllocationPreemptionPolicy {
+ private final ResourceThresholds overAllocationPreemptionThresholds;
+ private final ContainersMonitor containersMonitor;
+
+ public NMAllocationPreemptionPolicy(
+ ResourceThresholds preemptionThresholds,
+ ContainersMonitor containersMonitor) {
+ this.containersMonitor = containersMonitor;
+ this.overAllocationPreemptionThresholds = preemptionThresholds;
+ }
+
+ /**
+ * Get the amount of resources to reclaim by preempting opportunistic
+ * containers when over-allocation is turned on.
+ * @return the amount of resources to be reclaimed
+ */
+ public abstract ResourceUtilization getResourcesToReclaim();
+
+ public ContainersMonitor getContainersMonitor() {
+ return containersMonitor;
+ }
+
+ public ResourceThresholds getOverAllocationPreemptionThresholds() {
+ return overAllocationPreemptionThresholds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 7a7c78e..9be7574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -39,6 +39,13 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
+ * Get the total amount of resources allocated to running containers
+ * in terms of resource utilization.
+ * @return ResourceUtilization resource allocation
+ */
+ ResourceUtilization getTotalAllocation();
+
+ /**
* Get the amount of resources currently available to launch containers.
* @return Resource resources available to launch containers
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..188a108
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPreemptionPolicy.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+
+/**
+ * An implementation of {@link NMAllocationPreemptionPolicy} based on the
+ * snapshot of the latest containers utilization to determine how many
+ * resources need to be reclaimed by preempting opportunistic containers
+ * when over-allocation is turned on.
+ */
+public class SnapshotBasedOverAllocationPreemptionPolicy
+ extends NMAllocationPreemptionPolicy {
+ private final int absoluteMemoryPreemptionThresholdMb;
+ private final float cpuPreemptionThreshold;
+ private final int maxTimesCpuOverPreemption;
+ private int timesCpuOverPreemption;
+
+ public SnapshotBasedOverAllocationPreemptionPolicy(
+ ResourceThresholds preemptionThresholds,
+ int timesCpuOverPreemptionThreshold,
+ ContainersMonitor containersMonitor) {
+ super(preemptionThresholds, containersMonitor);
+ int memoryCapacityMb = (int)
+ (containersMonitor.getPmemAllocatedForContainers() / (1024 * 1024));
+ this.absoluteMemoryPreemptionThresholdMb = (int)
+ (preemptionThresholds.getMemoryThreshold() * memoryCapacityMb);
+ this.cpuPreemptionThreshold = preemptionThresholds.getCpuThreshold();
+ this.maxTimesCpuOverPreemption = timesCpuOverPreemptionThreshold;
+ }
+
+ @Override
+ public ResourceUtilization getResourcesToReclaim() {
+ ResourceUtilization utilization =
+ getContainersMonitor().getContainersUtilization(true).getUtilization();
+
+ int memoryOverLimit = utilization.getPhysicalMemory() -
+ absoluteMemoryPreemptionThresholdMb;
+ float vcoreOverLimit = utilization.getCPU() - cpuPreemptionThreshold;
+
+ if (vcoreOverLimit > 0) {
+ timesCpuOverPreemption++;
+ if (timesCpuOverPreemption > maxTimesCpuOverPreemption) {
+ timesCpuOverPreemption = 0;
+ } else {
+ // report no over limit for cpu if # of times CPU is over the preemption
+ // threshold is not greater the max number of times allowed
+ vcoreOverLimit = 0;
+ }
+ } else {
+ // reset the counter when cpu utilization goes under the preemption
+ // threshold before the max times allowed is reached
+ timesCpuOverPreemption = 0;
+ }
+
+ // sanitize so that zero is returned if the utilization is below
+ // the preemption threshold
+ vcoreOverLimit = Math.max(0, vcoreOverLimit);
+ memoryOverLimit = Math.max(0, memoryOverLimit);
+
+ return ResourceUtilization.newInstance(memoryOverLimit, 0, vcoreOverLimit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 92613ed..9f1506a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,7 +26,12 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -559,4 +564,125 @@ public abstract class BaseContainerManagerTest {
ContainerId.newContainerId(appAttemptId, cId);
return containerId;
}
+
+ /**
+ * A test implementation of {@link ContainersMonitor} that allows control of
+ * the current resource utilization.
+ */
+ protected static class ContainerMonitorForTest
+ extends ContainersMonitorImpl {
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, Time.now());
+ }
+
+ @Override
+ protected void checkOverAllocationPrerequisites() {
+ // do not check
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+
+ /**
+ * A test implementation of {@link ContainerManager} that allows its
+ * internal event queue to be drained for synchronization purpose,
+ * and an out-of-band check of the node resource utilization so that
+ * the node utilization can stay between the over-allocation threshold
+ * and the preemption threshold.
+ */
+ protected static class ContainerManagerForTest
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ public ContainerManagerForTest(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ @Override
+ protected AsyncDispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForTest(exec, dispatcher, context);
+ }
+
+ /**
+ * Check the node resource utilization out-of-band. If the utilization is
+ * below the over-allocation threshold, queued OPPORTUNISTIC containers
+ * will be launched to bring the node utilization up to the over-allocation
+ * threshold. If the utilization is above the preemption threshold, running
+ * OPPORTUNISTIC containers will be killed to bring the utilization down to
+ * the preemption threshold.
+ */
+ public void checkNodeResourceUtilization() {
+ ((ContainerMonitorForTest) getContainersMonitor()).checkUtilization();
+ drainAsyncEvents();
+ }
+
+ /**
+ * Drain the internal event queue.
+ */
+ public void drainAsyncEvents() {
+ ((DrainDispatcher) dispatcher).await();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 70066c6..d5c09b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -30,7 +30,6 @@ import java.util.concurrent.ConcurrentMap;
import com.google.common.base.Supplier;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse;
@@ -38,18 +37,15 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
@@ -65,8 +61,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
@@ -131,47 +125,8 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Override
protected ContainerManagerImpl createContainerManager(
DeletionService delSrvc) {
- return new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, dirsHandler) {
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainersMonitorImpl(exec, dispatcher, this.context) {
- // Define resources available for containers to be executed.
- @Override
- public long getPmemAllocatedForContainers() {
- return 2048 * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return 4;
- }
- };
- }
- };
+ return new ContainerManagerForTest(context, exec, delSrvc,
+ nodeStatusUpdater, metrics, dirsHandler, user);
}
@Override
@@ -393,6 +348,37 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
containerScheduler.getNumQueuedGuaranteedContainers());
Assert.assertEquals(2,
containerScheduler.getNumQueuedOpportunisticContainers());
+
+ // we have just one container that requested 2048 MB of memory and 1 vcore
+ // running, its resource utilization is zero.
+ // check the node resource utilization and the two OPPORTUNISTIC containers
+ // that are being queued should stay in the queue because over-allocation
+ // is turn off.
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ ContainerId containerId0 = createContainerId(0);
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(containerId0)) {
+ Assert.assertEquals(ContainerSubState.RUNNING,
+ status.getContainerSubState());
+ } else {
+ Assert.assertEquals(ContainerSubState.SCHEDULED,
+ status.getContainerSubState());
+ }
+ }
+ containerScheduler = containerManager.getContainerScheduler();
+ // Ensure two containers are properly queued.
+ Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
+ Assert.assertEquals(0,
+ containerScheduler.getNumQueuedGuaranteedContainers());
+ Assert.assertEquals(2,
+ containerScheduler.getNumQueuedOpportunisticContainers());
}
/**
@@ -476,6 +462,91 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
/**
+ * Start two OPPORTUNISTIC containers which together ask for all
+ * the allocations available on the node. When the node resource
+ * utilization goes over the preemption thresholds, neither of
+ * the containers should be preempted because there is no
+ * over-allocation at the moment and they can safely use up all
+ * the resources availabe on the node.
+ */
+ @Test
+ public void testNoOpportunisticContainerPreemptionUponHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ // start two OPPORTUNISTIC containers that together takes up
+ // all allocations of the node. They two can be launched immediately
+ // because there is enough free allocation. When they uses up
+ // all their resource allocations, that is, the node is fully
+ // utilized, none of the OPPORTUNISTIC containers shall be killed
+ // because the node is not being over-allocated
+ List<StartContainerRequest> list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ recordFactory.newRecordInstance(ContainerLaunchContext.class),
+ createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(1024, 2),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+ list.add(StartContainerRequest.newInstance(
+ recordFactory.newRecordInstance(ContainerLaunchContext.class),
+ createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(1024, 2),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+
+ // the two OPPORTUNISTIC containers shall be launched immediately
+ // because there is just enough allocation to launch them both.
+ BaseContainerManagerTest.waitForContainerState(containerManager,
+ createContainerId(0),
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
+ BaseContainerManagerTest.waitForContainerState(containerManager,
+ createContainerId(1),
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
+
+ // Ensure all containers are running.
+ List<ContainerId> statList = new ArrayList<>();
+ for (int i = 0; i < 2; i++) {
+ statList.add(createContainerId(i));
+ }
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+
+ // we have two containers running, both of which are using all of
+ // their allocations. The node is being fully utilized in terms
+ // of both memory and CPU
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // the two running OPPORTUNISTIC containers shall continue to run
+ // because when the node is not be over-allocated, it is safe to
+ // let the containers use up all the resources, no OPPORTUNISTIC
+ // containers shall be preempted
+ containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+ }
+
+ /**
* Submit two OPPORTUNISTIC and one GUARANTEED containers. The resources
* requests by each container as such that only one can run in parallel.
* Thus, the OPPORTUNISTIC container that started running, will be
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/24] hadoop git commit: YARN-6921. Allow resource request to opt
out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6921. Allow resource request to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d106fdbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d106fdbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d106fdbe
Branch: refs/heads/YARN-1011
Commit: d106fdbe7afb82c3d5951cabe854c409721477e9
Parents: ff73fb9
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:03:05 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:13 2018 -0700
----------------------------------------------------------------------
.../scheduler/common/PendingAsk.java | 15 +++-
.../scheduler/fair/FSAppAttempt.java | 5 ++
.../LocalityAppPlacementAllocator.java | 27 ++++++-
.../scheduler/fair/FairSchedulerTestBase.java | 40 ++++++++--
.../scheduler/fair/TestFairScheduler.java | 77 ++++++++++++++++++++
5 files changed, 153 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d106fdbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
index 2ed3e83..470dbbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
@@ -30,16 +30,21 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class PendingAsk {
private final Resource perAllocationResource;
private final int count;
- public final static PendingAsk ZERO = new PendingAsk(Resources.none(), 0);
+ public final static PendingAsk ZERO =
+ new PendingAsk(Resources.none(), 0, false);
+
+ private final boolean isGuaranteedTypeEnforced;
public PendingAsk(ResourceSizing sizing) {
this.perAllocationResource = sizing.getResources();
this.count = sizing.getNumAllocations();
+ this.isGuaranteedTypeEnforced = true;
}
- public PendingAsk(Resource res, int num) {
+ public PendingAsk(Resource res, int num, boolean guaranteedTypeEnforced) {
this.perAllocationResource = res;
this.count = num;
+ this.isGuaranteedTypeEnforced = guaranteedTypeEnforced;
}
public Resource getPerAllocationResource() {
@@ -50,11 +55,17 @@ public class PendingAsk {
return count;
}
+ public boolean isGuaranteedTypeEnforced() {
+ return isGuaranteedTypeEnforced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("<per-allocation-resource=");
sb.append(getPerAllocationResource());
+ sb.append(", isGuaranteedEnforced=");
+ sb.append(isGuaranteedTypeEnforced());
sb.append(",repeat=");
sb.append(getCount());
sb.append(">");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d106fdbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 1928591..0effd42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -859,6 +859,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
boolean reserved, boolean opportunistic,
SchedulerRequestKey schedulerKey) {
+ if (pendingAsk.isGuaranteedTypeEnforced() && opportunistic) {
+ // do not attempt to assign an OPPORTUNISTIC container to a resource
+ // request that has explicitly opted out of oversubscription
+ return Resources.none();
+ }
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d106fdbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index 4557350..68d53e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
@@ -188,13 +190,16 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
PendingAsk lastPendingAsk =
lastRequest == null ? null : new PendingAsk(
- lastRequest.getCapability(), lastRequest.getNumContainers());
+ lastRequest.getCapability(), lastRequest.getNumContainers(),
+ enforceGuaranteedExecutionType(lastRequest));
String lastRequestedNodePartition =
lastRequest == null ? null : lastRequest.getNodeLabelExpression();
updateResult = new PendingAskUpdateResult(lastPendingAsk,
new PendingAsk(request.getCapability(),
- request.getNumContainers()), lastRequestedNodePartition,
+ request.getNumContainers(),
+ enforceGuaranteedExecutionType(request)),
+ lastRequestedNodePartition,
request.getNodeLabelExpression());
}
}
@@ -234,8 +239,9 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
if (null == request) {
return PendingAsk.ZERO;
} else{
+ boolean guaranteedEnforced = enforceGuaranteedExecutionType(request);
return new PendingAsk(request.getCapability(),
- request.getNumContainers());
+ request.getNumContainers(), guaranteedEnforced);
}
} finally {
readLock.unlock();
@@ -243,6 +249,21 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
}
+ /**
+ * Check for a given ResourceRequest, if its guaranteed execution type
+ * needs to be enforced.
+ * @param request resource request
+ * @return true if its guaranteed execution type is to be enforced.
+ * false otherwise
+ */
+ private static boolean enforceGuaranteedExecutionType(
+ ResourceRequest request) {
+ ExecutionTypeRequest executionType = request.getExecutionTypeRequest();
+ return executionType != null &&
+ executionType.getExecutionType() == ExecutionType.GUARANTEED &&
+ executionType.getEnforceExecutionType();
+ }
+
@Override
public int getOutstandingAsksCount(String resourceName) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d106fdbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 3ac3849..43a3931 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -117,9 +119,15 @@ public class FairSchedulerTestBase {
relaxLocality);
}
+ protected ResourceRequest createResourceRequest(int memory, int vcores,
+ String host, int priority, int numContainers, boolean relaxLocality) {
+ return createResourceRequest(memory, vcores, host, priority,
+ numContainers, relaxLocality, false);
+ }
+
protected ResourceRequest createResourceRequest(
int memory, int vcores, String host, int priority, int numContainers,
- boolean relaxLocality) {
+ boolean relaxLocality, boolean guaranteedExecutionEnforced) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setCapability(BuilderUtils.newResource(memory, vcores));
request.setResourceName(host);
@@ -129,6 +137,11 @@ public class FairSchedulerTestBase {
request.setPriority(prio);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ if (guaranteedExecutionEnforced) {
+ ExecutionTypeRequest executionType = ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true);
+ request.setExecutionTypeRequest(executionType);
+ }
return request;
}
@@ -152,6 +165,13 @@ public class FairSchedulerTestBase {
}
protected ApplicationAttemptId createSchedulingRequest(
+ int memory, String queueId, String userId,
+ int numContainers, boolean guaranteedExecutionEnforced) {
+ return createSchedulingRequest(memory, 1, queueId,
+ userId, numContainers, 1, guaranteedExecutionEnforced);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers) {
return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1);
}
@@ -165,16 +185,24 @@ public class FairSchedulerTestBase {
protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers,
int priority) {
- ResourceRequest request = createResourceRequest(memory, vcores,
- ResourceRequest.ANY, priority, numContainers, true);
+ return createSchedulingRequest(memory, vcores, queueId,
+ userId, numContainers, priority, false);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
+ int memory, int vcores, String queueId, String userId, int numContainers,
+ int priority, boolean guaranteedExecutionEnforced) {
+ ResourceRequest request = createResourceRequest(
+ memory, vcores, ResourceRequest.ANY, priority,
+ numContainers, true, guaranteedExecutionEnforced);
return createSchedulingRequest(Lists.newArrayList(request), queueId,
- userId);
+ userId);
}
protected ApplicationAttemptId createSchedulingRequest(
Collection<ResourceRequest> requests, String queueId, String userId) {
- ApplicationAttemptId id =
- createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
+ ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
+ this.ATTEMPT_ID++);
scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d106fdbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ff620b8..fbb7243 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2712,6 +2712,83 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getPriority().getPriority());
}
+ @Test
+ public void testResourceRequestOptOutOfOversubscription() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription and
+ // asks for more than what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/24] hadoop git commit: YARN-6672. Add NM preemption of
opportunistic containers when utilization goes high.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
index 183d868..a2d4aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -20,12 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -36,11 +33,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
@@ -53,8 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
@@ -78,8 +70,6 @@ import java.util.concurrent.ConcurrentHashMap;
public class TestContainerSchedulerWithOverAllocation
extends BaseContainerManagerTest {
private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
- private static final int NM_CONTAINERS_VCORES = 4;
- private static final int NM_CONTAINERS_MEMORY_MB = 2048;
static {
LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
@@ -115,6 +105,11 @@ public class TestContainerSchedulerWithOverAllocation
conf.setFloat(
YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
0.75f);
+ conf.setFloat(YarnConfiguration.NM_OVERALLOCATION_CPU_PREEMPTION_THRESHOLD,
+ 0.8f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_PREEMPTION_THRESHOLD, 0.8f);
+ conf.setInt(YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_CPU_COUNT, 2);
// disable the monitor thread in ContainersMonitor to allow control over
// when opportunistic containers are launched with over-allocation
conf.setBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
@@ -134,9 +129,9 @@ public class TestContainerSchedulerWithOverAllocation
StartContainersRequest allRequests = StartContainersRequest.newInstance(
new ArrayList<StartContainerRequest>() { {
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
} }
);
containerManager.startContainers(allRequests);
@@ -171,9 +166,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
+ BuilderUtils.newResource(824, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -191,11 +186,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// this container is not expected to be started immediately because
// opportunistic containers cannot be started if the node would be
@@ -212,8 +205,8 @@ public class TestContainerSchedulerWithOverAllocation
});
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container is expected to be started immediately because there
// are (memory: 1024, vcore: 0.625) available based on over-allocation
@@ -247,9 +240,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
+ BuilderUtils.newResource(824, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -267,17 +260,14 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
// try to start opportunistic containers out of band because they can
// not be launched at container scheduler event if the node would be
// over-allocated.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container will not start immediately because there is not
// enough resource available at the moment either in terms of
@@ -312,9 +302,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -331,11 +321,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC))
));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// this container is not expected to be started immediately because
// opportunistic containers cannot be started if the node would be
@@ -352,8 +340,8 @@ public class TestContainerSchedulerWithOverAllocation
});
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// this container is expected to be started because there is resources
// available because the actual utilization is very low
@@ -384,9 +372,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -406,13 +394,11 @@ public class TestContainerSchedulerWithOverAllocation
for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
moreContainerRequests.add(
createStartContainerRequest(2 + a,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
}
containerManager.startContainers(
StartContainersRequest.newInstance(moreContainerRequests));
-
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// All OPPORTUNISTIC containers but the last one should be queued.
// The last OPPORTUNISTIC container to launch should be killed.
@@ -454,9 +440,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1200, 1), true));
+ BuilderUtils.newResource(1200, 1), ExecutionType.GUARANTEED));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(400, 1), true));
+ BuilderUtils.newResource(400, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -473,7 +459,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(400, 1), false))
+ BuilderUtils.newResource(400, 1), ExecutionType.OPPORTUNISTIC))
));
// the OPPORTUNISTIC container can be safely launched even though
@@ -507,9 +493,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
+ BuilderUtils.newResource(824, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -527,7 +515,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
// the GUARANTEED container is expected be launched immediately without
@@ -561,9 +549,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
+ BuilderUtils.newResource(824, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -581,7 +571,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -631,16 +621,18 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), false));
+ BuilderUtils.newResource(1024, 1),
+ ExecutionType.OPPORTUNISTIC));
}
}
));
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// Two OPPORTUNISTIC containers are expected to start with the
// unallocated resources, but one will be queued because no
@@ -655,8 +647,8 @@ public class TestContainerSchedulerWithOverAllocation
// try to start the opportunistic container out of band because it can
// not be launched at container scheduler event if the node would be
// over-allocated.
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// now the queued opportunistic container should also start
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -671,7 +663,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), true))
+ BuilderUtils.newResource(512, 1), ExecutionType.GUARANTEED))
));
// the GUARANTEED container is expected be launched immediately without
@@ -690,83 +682,6 @@ public class TestContainerSchedulerWithOverAllocation
}
/**
- * Start four OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node. The real resource utilization of the first two
- * OPPORTUNISTIC containers are high whereas that of the latter two are
- * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
- * container will eventually start running after preempting the third
- * and fourth OPPORTUNISTIC container (which releases no resources) and
- * then the second OPPORTUNISTIC container.
- */
- public void
- testKillOppContainersConservativelyWithOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All four GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- // the containers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start a GUARANTEED container when there's nothing left unallocated
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(4,
- BuilderUtils.newResource(1024, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- add(createContainerId(3));
- add(createContainerId(4));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(0)) ||
- status.getContainerId().equals(createContainerId(4))) {
- Assert.assertEquals(
- ContainerSubState.RUNNING, status.getContainerSubState());
- } else {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
- /**
* Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
* which in aggregate exceeds the capacity of the node. The first two
* OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
@@ -784,11 +699,11 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(0,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), true));
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
}
}
));
@@ -810,9 +725,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(4,
- BuilderUtils.newResource(800, 1), false));
+ BuilderUtils.newResource(800, 1), ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -831,8 +746,7 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.DONE);
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
// only one OPPORTUNISTIC container is start because no over-allocation
// is allowed to start OPPORTUNISTIC containers at container finish event.
@@ -854,10 +768,8 @@ public class TestContainerSchedulerWithOverAllocation
// now try to start the OPPORTUNISTIC container that was queued because
// we don't start OPPORTUNISTIC containers at container finish event if
// the node would be over-allocated
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .startContainersOutOfBandUponLowUtilization();
- ((LongRunningContainerSimulatingContainersManager) containerManager)
- .drainAsyncEvents();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(4), ContainerSubState.RUNNING);
verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
@@ -884,7 +796,7 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(StartContainersRequest.newInstance(
Collections.singletonList(
createStartContainerRequest(0,
- BuilderUtils.newResource(2048, 4), true))));
+ BuilderUtils.newResource(2048, 4), ExecutionType.GUARANTEED))));
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(0), ContainerSubState.RUNNING);
@@ -897,9 +809,9 @@ public class TestContainerSchedulerWithOverAllocation
new ArrayList<StartContainerRequest>() {
{
add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
}
}
));
@@ -913,8 +825,8 @@ public class TestContainerSchedulerWithOverAllocation
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
// try to start opportunistic containers out of band.
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// no containers in queue are expected to be launched because the
// containers utilization is not below the over-allocation threshold
@@ -934,8 +846,8 @@ public class TestContainerSchedulerWithOverAllocation
setContainerResourceUtilization(
ResourceUtilization.newInstance(512, 0, 1.0f/8));
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
// the two OPPORTUNISTIC containers are expected to be launched
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -952,11 +864,332 @@ public class TestContainerSchedulerWithOverAllocation
});
}
+ /**
+ * Start a GUARANTEED container, an OPPORTUNISTIC container, a GUARANTEED
+ * container and another OPPORTUNISTIC container in order. When the node
+ * memory utilization is over its preemption threshold, the two OPPORTUNISTIC
+ * containers should be killed.
+ */
+ @Test
+ public void testPreemptOpportunisticContainersUponHighMemoryUtilization()
+ throws Exception {
+ containerManager.start();
+
+ // try to start four containers at once. the first GUARANTEED container
+ // that requests (1024 MB, 1 vcore) can be launched because there is
+ // (2048 MB, 4 vcores) unallocated. The second container, which is
+ // OPPORTUNISTIC, can also be launched because it asks for 512 MB, 1 vcore
+ // which is less than what is left unallocated after launching the first
+ // one GUARANTEED container, (1024 MB, 3 vcores).
+ // The 3rd container, which is GUARANTEED, can also be launched because
+ // the node resource utilization utilization is zero such that
+ // over-allocation kicks in. The 4th one, an OPPORTUNISTIC container,
+ // will be queued because OPPORTUNISTIC containers can only be
+ // launched when node resource utilization is checked, if launching them
+ // would cause node over-allocation.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(300, 1), ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
+
+ // the first three containers are all expected to start
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.SCHEDULED);
+
+ // try to check node resource utilization and start the second
+ // opportunistic containers out of band. Because the node resource
+ // utilization is zero at the moment, over-allocation will kick in
+ // and the container will be launched.
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers memory utilization is over the preemption threshold
+ // (2048 > 2048 * 0.8 = 1638.4)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 0.5f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // (2048 - 2048 * 0.8) = 409.6 MB of memory needs to be reclaimed,
+ // which shall result in both OPPORTUNISTIC containers to be preempted.
+ // (Preempting the most recently launched OPPORTUNISTIC container, that
+ // is the 4th container, would only release at most 300 MB of memory)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.DONE);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.DONE);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ }
+
+ /**
+ * Start a GUARANTEED container followed by an OPPORTUNISTIC container, which
+ * in aggregates does not take more than the capacity of the node.
+ * When the node memory utilization is above the preemption threshold, the
+ * OPPORTUNISTIC container should not be killed because the node is not being
+ * over-allocated.
+ */
+ @Test
+ public void testNoPreemptionUponHighMemoryUtilizationButNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ // start two containers, one GUARANTEED and one OPPORTUNISTIC, that together
+ // take up all the allocations (2048 MB of memory and 4 vcores available on
+ // the node). They can be both launched immediately because there are enough
+ // allocations to do so. When the two containers fully utilize their
+ // resource requests, that is, the node is being 100% utilized, the
+ // OPPORTUNISTIC container shall continue to run because the node is
+ // not be over-allocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 2),
+ ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 2),
+ ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ // both containers shall be launched immediately because there are
+ // enough allocations to do so
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the node is being fully utilized, which is above the preemption
+ // threshold (2048 * 0.75 = 1536 MB, 1.0f)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // no containers shall be preempted because the node is not being
+ // over-allocated so it is safe to allow the node to be fully utilized
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start a GUARANTEED container, an OPPORTUNISTIC container, a GUARANTEED
+ * container and another OPPORTUNISTIC container in order. When the node
+ * cpu utilization is over its preemption threshold a few times in a row,
+ * the two OPPORTUNISTIC containers should be killed one by one.
+ */
+ @Test
+ public void testPreemptionUponHighCPUUtilization() throws Exception {
+ containerManager.start();
+
+ // try to start 4 containers at once. The first container, can be
+ // safely launched immediately (2048 MB, 4 vcores left unallocated).
+ // The second container, can also be launched immediately, because
+ // there is enough resources unallocated after launching the first
+ // container (2048 - 512 = 1536 MB, 4 - 2 = 2 vcores). After launching
+ // the first two containers, there are 1024 MBs of memory and 1 vcore
+ // left unallocated, so there is not enough allocation to launch the
+ // third container. But because the third container is GUARANTEED and
+ // the node resource utilization is zero, we can launch it based on
+ // over-allocation (the projected resource utilization will be 512 MB
+ // of memory and 2 vcores, below the over-allocation threshold)
+ // The fourth container, which is OPPORTUNISTIC, will be queued because
+ // OPPORTUNISTIC containers can not be launched based on over-allocation
+ // upon container start requests (they can only be launched when node
+ // resource utilization is checked in ContainersMonitor)
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 2), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 2), ExecutionType.GUARANTEED));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), ExecutionType.OPPORTUNISTIC));
+ }
+ }
+ ));
+ ((ContainerManagerForTest) containerManager).drainAsyncEvents();
+ // the first three containers are expected to start. The first two
+ // can be launched based on free allocation, the third can be
+ // launched based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // try to start second opportunistic containers out of band.
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // the second opportunistic container is expected to start because
+ // the node resource utilization is at zero, the projected utilization
+ // is 512 MBs of memory and 1 vcore
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ final float fullCpuUtilization = 1.0f;
+
+ // the containers CPU utilization is over its preemption threshold (0.8f)
+ // for the first time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // all containers should continue to be running because we don't
+ // preempt OPPORTUNISTIC containers right away
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+
+ // the containers CPU utilization is over its preemption threshold (0.8f)
+ // for the second time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // all containers should continue to be running because we don't preempt
+ // OPPORTUNISTIC containers when the cpu is over the preemption threshold
+ // (0.8f) the second time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold (0.8f)
+ // for the third time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // because CPU utilization is over its preemption threshold three times
+ // consecutively, the amount of cpu utilization over the preemption
+ // threshold, that is, 1.0 - 0.8 = 0.2f CPU needs to be reclaimed and
+ // as a result, the most recently launched OPPORTUNISTIC container should
+ // be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.DONE);
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // again, the containers CPU utilization is over the preemption threshold
+ // (0.8f) for the first time (the cpu over-limit count is reset every time
+ // a preemption is triggered)
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // no CPU resource is expected to be reclaimed when the CPU utilization
+ // goes over the preemption threshold the first time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold (0.9f)
+ // for the second time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // still no CPU resource is expected to be reclaimed when the CPU
+ // utilization goes over the preemption threshold the second time
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+
+ // the containers CPU utilization is over the preemption threshold
+ // for the third time
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, fullCpuUtilization));
+ ((ContainerManagerForTest) containerManager)
+ .checkNodeResourceUtilization();
+
+ // because CPU utilization is over its preemption threshold three times
+ // consecutively, the amount of cpu utilization over the preemption
+ // threshold, that is, 1.0 - 0.8 = 0.2f CPU needs to be reclaimed and
+ // as a result, the other OPPORTUNISTIC container should be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.DONE);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.DONE);
+ }
+ });
+ }
+
private void setContainerResourceUtilization(ResourceUtilization usage) {
- ((ContainerMonitorForOverallocationTest)
- containerManager.getContainersMonitor())
- .setContainerResourceUsage(usage);
+ ((ContainerMonitorForTest) containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
}
private void allowContainerToSucceed(int containerId) {
@@ -965,12 +1198,11 @@ public class TestContainerSchedulerWithOverAllocation
}
- protected StartContainerRequest createStartContainerRequest(int containerId,
- Resource resource, boolean isGuaranteed) throws IOException {
+ protected StartContainerRequest createStartContainerRequest(
+ int containerId, Resource resource, ExecutionType executionType)
+ throws IOException {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
- ExecutionType.OPPORTUNISTIC;
Token containerToken = createContainerToken(
createContainerId(containerId),
DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
@@ -1004,9 +1236,7 @@ public class TestContainerSchedulerWithOverAllocation
* container processes for testing purposes.
*/
private static class LongRunningContainerSimulatingContainersManager
- extends ContainerManagerImpl {
-
- private final String user;
+ extends ContainerManagerForTest {
LongRunningContainerSimulatingContainersManager(
Context context, ContainerExecutor exec,
@@ -1014,27 +1244,8 @@ public class TestContainerSchedulerWithOverAllocation
NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics,
LocalDirsHandlerService dirsHandler, String user) {
- super(context, exec, deletionContext,
- nodeStatusUpdater, metrics, dirsHandler);
- this.user = user;
- }
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- @Override
- protected AsyncDispatcher createDispatcher() {
- return new DrainDispatcher();
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler, user);
}
/**
@@ -1060,27 +1271,10 @@ public class TestContainerSchedulerWithOverAllocation
throws Exception {
return "123";
}
-
};
}
};
}
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainerMonitorForOverallocationTest(exec,
- dispatcher, context);
- }
-
- public void startContainersOutOfBandUponLowUtilization() {
- ((ContainerMonitorForOverallocationTest) getContainersMonitor())
- .attemptToStartContainersUponLowUtilization();
- }
-
- public void drainAsyncEvents() {
- ((DrainDispatcher) dispatcher).await();
- }
}
/**
@@ -1176,56 +1370,4 @@ public class TestContainerSchedulerWithOverAllocation
}
}
}
-
- /**
- * A test implementation of container monitor that allows control of
- * current resource utilization.
- */
- private static class ContainerMonitorForOverallocationTest
- extends ContainersMonitorImpl {
-
- private ResourceUtilization containerResourceUsage =
- ResourceUtilization.newInstance(0, 0, 0.0f);
-
- ContainerMonitorForOverallocationTest(ContainerExecutor exec,
- AsyncDispatcher dispatcher, Context context) {
- super(exec, dispatcher, context);
- }
-
- @Override
- public long getPmemAllocatedForContainers() {
- return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return NM_CONTAINERS_VCORES;
- }
-
- @Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- return new ContainersMonitor.ContainersResourceUtilization(
- containerResourceUsage, System.currentTimeMillis());
- }
-
- @Override
- protected void checkOverAllocationPrerequisites() {
- // do not check
- }
-
-
- public void setContainerResourceUsage(
- ResourceUtilization containerResourceUsage) {
- this.containerResourceUsage = containerResourceUsage;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a54014d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
new file mode 100644
index 0000000..bbc7c49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestSnapshotBasedOverAllocationPreemptionPolicy.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link SnapshotBasedOverAllocationPreemptionPolicy}.
+ */
+public class TestSnapshotBasedOverAllocationPreemptionPolicy {
+ // Both the CPU preemption threshold and the memory preemption threshold
+ // are 75%
+ private final static ResourceThresholds PREEMPTION_THRESHOLDS =
+ ResourceThresholds.newInstance(0.75f, 0.75f);
+
+ // The CPU utilization is allowed to go over the cpu preemption threshold
+ // 2 times in a row before any container is preempted to reclaim cpu resources
+ private final static int MAX_CPU_OVER_PREEMPTION_THRESHOLDS = 2;
+
+ private final ContainersMonitor containersMonitor =
+ mock(ContainersMonitor.class);
+
+ @Before
+ public void setUp() {
+ // the node has an allocation of 2048 MB of memory
+ when(containersMonitor.getPmemAllocatedForContainers()).
+ thenReturn(2048 * 1024 * 1024L);
+ }
+
+ /**
+ * The memory utilization goes above its preemption threshold,
+ * 2048 * 0.75f = 1536 MB (the node has an allocation of 2048 MB memory).
+ */
+ @Test
+ public void testMemoryOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current memory utilization, 2000 MB is over the preemption
+ // threshold, 2048 * 0.75, which is 1536 MB. The CPU utilization,
+ // 0.5f is below the preemption threshold, 0.75f.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 0.5f),
+ Time.now()));
+
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * The CPU utilization goes above its preemption threshold, 0.75f.
+ */
+ @Test
+ public void testCpuOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is also below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 0.5f),
+ Time.now()));
+ // no resources shall be reclaimed
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is again over the preemption
+ // threshold, 0.75f. In the meantime the memory utilization, 1000 MB
+ // is below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the second time in a row
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // the third time in a row. In the meantime the memory utilization, 1000 MB
+ // is below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // the amount of cpu utilization over the preemption threshold, that is,
+ // 1.0 - 0.75f = 0.25, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.25f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * Both memory and CPU utilization go over their preemption thresholds
+ * respectively.
+ */
+ @Test
+ public void testMemoryCpuOverPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. The memory utilization, 2000 MB, however, is above the memory
+ // preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 0.5f),
+ Time.now()));
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the preemption threshold,
+ // 0.75f, for the first time. The memory utilization, 1000 MB is below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the first time.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is again over the preemption
+ // threshold, 0.75f. In the meantime the memory utilization, 1000 MB
+ // is still below the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 1.0f),
+ Time.now()));
+ // no resources shall be reclaimed because the cpu utilization is allowed
+ // to go over the preemption threshold at most two times in a row. It is
+ // just over the preemption threshold for the second time in a row.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0.0f),
+ preemptionPolicy.getResourcesToReclaim());
+
+ // the current CPU utilization, 1.0f, is over the CPU preemption threshold,
+ // 0.75f, the third time in a row. In the meantime, the memory utilization,
+ // 2000 MB, is also over the memory preemption threshold,
+ // 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(2000, 0, 1.0f),
+ Time.now()));
+ // the amount of memory utilization over the preemption threshold, that is,
+ // 2000 - (2048 * 0.75) = 464 MB of memory, and the amount of cpu
+ // utilization over the preemption threshold, that is, 1.0f - 0.75f = 0.25f,
+ // shall be reclaimed.
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(464, 0, 0.25f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+
+ /**
+ * Both memory and CPU utilization are under their preemption thresholds.
+ */
+ @Test
+ public void testBothMemoryAndCpuUnderPreemptionThreshold() {
+ SnapshotBasedOverAllocationPreemptionPolicy preemptionPolicy =
+ new SnapshotBasedOverAllocationPreemptionPolicy(PREEMPTION_THRESHOLDS,
+ MAX_CPU_OVER_PREEMPTION_THRESHOLDS, containersMonitor);
+
+ // the current CPU utilization, 0.5f, is below the preemption threshold,
+ // 0.75f. In the meantime the memory utilization, 1000 MB is also below
+ // the memory preemption threshold, 2048 * 0.75 = 1536 MB.
+ when(containersMonitor.getContainersUtilization(anyBoolean())).thenReturn(
+ new ContainersMonitor.ContainersResourceUtilization(
+ ResourceUtilization.newInstance(1000, 0, 0.5f),
+ Time.now()));
+ // no resources shall be reclaimed because both CPU and memory utilization
+ // are under the preemption threshold
+ Assert.assertEquals(
+ ResourceUtilization.newInstance(0, 0, 0f),
+ preemptionPolicy.getResourcesToReclaim());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/24] hadoop git commit: Revert "Revert "YARN-6675. Add NM support
to launch opportunistic containers based on overallocation. Contributed by
Haibo Chen.""
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index 6b3ac67..f24df38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -66,8 +67,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceUtilizationTracker
- allocationBasedResourceUtilizationTracker;
+ @Mock private AllocationBasedResourceTracker
+ allocationBasedResourceTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -85,12 +86,13 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
+ when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceUtilizationTracker)
- .addContainerResources(container);
+ doNothing().when(allocationBasedResourceTracker)
+ .containerLaunched(container);
}
@After public void tearDown() {
@@ -112,8 +114,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -132,8 +134,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -152,8 +154,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -172,8 +174,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -192,8 +194,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -212,8 +214,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -232,8 +234,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -252,8 +254,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -272,8 +274,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -292,8 +294,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -311,8 +313,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -330,7 +332,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9db0368/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
new file mode 100644
index 0000000..384b116
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -0,0 +1,1121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+/**
+ * Test ContainerScheduler behaviors when NM overallocation is turned on.
+ */
+public class TestContainerSchedulerWithOverAllocation
+ extends BaseContainerManagerTest {
+ private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ static {
+ LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
+ }
+
+ public TestContainerSchedulerWithOverAllocation()
+ throws UnsupportedFileSystemException {
+ }
+
+ @Override
+ protected ContainerExecutor createContainerExecutor() {
+ DefaultContainerExecutor exec =
+ new LongRunningContainerSimulatingContainerExecutor();
+ exec.setConf(conf);
+ return exec;
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(
+ DeletionService delSrvc) {
+ return new LongRunningContainerSimulatingContainersManager(
+ context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
+ }
+
+ @Override
+ public void setup() throws IOException {
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
+ NM_OPPORTUNISTIC_QUEUE_LIMIT);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ 0.75f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ 0.75f);
+ super.setup();
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
+ * not exceed the capacity of the node. Both containers are expected to start
+ * running immediately.
+ */
+ @Test
+ public void testStartMultipleContainersWithoutOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ StartContainersRequest allRequests = StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() { {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ } }
+ );
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
+ * is very low relative to their resource request, resulting in a low node
+ * utilization. Then start another OPPORTUNISTIC containers which requests
+ * more than what's left unallocated on the node. Due to overallocation
+ * being turned on and node utilization being low, the second OPPORTUNISTIC
+ * container is also expected to be launched immediately.
+ */
+ @Test
+ public void testStartOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started immediately because there
+ // are (memory: 1024, vcore: 0.625) available based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
+ * of the resources they requested, resulting in a high node utilization.
+ * Then start another OPPORTUNISTIC containers which requests more than what's
+ * left unallocated on the node. Because of the high resource utilization on
+ * the node, the projected utilization, if we were to start the second
+ * OPPORTUNISTIC container immediately, will go over the NM overallocation
+ * threshold, so the second OPPORTUNISTIC container is expected to be queued.
+ */
+ @Test
+ public void testQueueOppContainerWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1500, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+ // this container will not start immediately because there is not
+ // enough resource available at the moment either in terms of
+ // resources unallocated or in terms of the actual utilization
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+ }
+
+ /**
+ * Start two GUARANTEED containers which in aggregate takes up the whole node
+ * capacity, yet whose utilization is low relative to their resource request,
+ * resulting in a low node resource utilization. Then try to start another
+ * OPPORTUNISTIC containers. Because the resource utilization across the node
+ * is low and overallocation being turned on, the OPPORTUNISTIC container is
+ * expected to be launched immediately even though there is no resources left
+ * unallocated.
+ */
+ @Test
+ public void testStartOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(800, 0, 1.0f/8));
+
+ // start a container when there is no resources left unallocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started because there is resources
+ // available because the actual utilization is very low
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ /**
+ * Start two GUARANTEED containers which in aggregate take up the whole node
+ * capacity and fully utilize the resources they requested. Then try to start
+ * four OPPORTUNISTIC containers of which three will be queued and one will be
+ * killed because of the max queue length is 3.
+ */
+ @Test
+ public void testQueueOppContainersWithFullUtilization() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers are fully utilizing their resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f/8));
+
+ // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
+ // queue can hold when there is no unallocated resource left.
+ List<StartContainerRequest> moreContainerRequests =
+ new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
+ for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
+ moreContainerRequests.add(
+ createStartContainerRequest(2 + a,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ containerManager.startContainers(
+ StartContainersRequest.newInstance(moreContainerRequests));
+
+ // All OPPORTUNISTIC containers but the last one should be queued.
+ // The last OPPORTUNISTIC container to launch should be killed.
+ BaseContainerManagerTest.waitForContainerState(
+ containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
+ ContainerState.COMPLETE);
+
+ HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
+ new HashMap<>();
+ expectedContainerStatus.put(
+ createContainerId(0), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(1), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
+ ContainerSubState.DONE);
+ for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
+ expectedContainerStatus.put(
+ createContainerId(i + 2), ContainerSubState.SCHEDULED);
+ }
+ verifyContainerStatuses(expectedContainerStatus);
+ }
+
+ /**
+ * Start two GUARANTEED containers that together does not take up the
+ * whole node. Then try to start one OPPORTUNISTIC container that will
+ * fit into the remaining unallocated space on the node.
+ * The OPPORTUNISTIC container is expected to start even though the
+ * current node utilization is above the NM overallocation threshold,
+ * because it's always safe to launch containers as long as the node
+ * has not been fully allocated.
+ */
+ @Test
+ public void testStartOppContainerWithHighUtilizationNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1200, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(400, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is above the over-allocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1600, 0, 1.0f/2));
+
+ // start a container that can just fit in the remaining unallocated space
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(400, 1), false))
+ ));
+
+ // the OPPORTUNISTIC container can be safely launched even though
+ // the container utilization is above the NM overallocation threshold
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers first whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization. Then
+ * try to start a GUARANTEED container which requests more than what's left
+ * unallocated on the node. Because the node utilization is low and NM
+ * overallocation is turned on, the GUARANTEED container is expected to be
+ * started immediately without killing any running OPPORTUNISTIC containers.
+ */
+ @Test
+ public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers.
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers whose utilization will be high relative
+ * to the resources they requested, resulting in a high node utilization.
+ * Then try to start a GUARANTEED container which requests more than what's
+ * left unallocated on the node. Because the node is under high utilization,
+ * the second OPPORTUNISTIC container is expected to be killed in order to
+ * make room for the GUARANTEED container.
+ */
+ @Test
+ public void testKillOppContainersWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is very high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1800, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ // the last launched OPPORTUNISTIC container is expected to be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ } else {
+ Assert.assertEquals(status.getContainerId() + " is not RUNNING",
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+
+ /**
+ * Start three OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node, yet whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization.
+ * Then try to start a GUARANTEED container. Even though the node has
+ * nothing left unallocated, it is expected to start immediately
+ * without killing any running OPPORTUNISTIC containers because the node
+ * utilization is very low and overallocation is turned on.
+ */
+ @Test
+ public void testKillNoOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All three GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1024, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start four OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node. The real resource utilization of the first two
+ * OPPORTUNISTIC containers are high whereas that of the latter two are
+ * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
+ * container will eventually start running after preempting the third
+ * and fourth OPPORTUNISTIC container (which releases no resources) and
+ * then the second OPPORTUNISTIC container.
+ */
+ @Test
+ public void
+ testKillOppContainersConservativelyWithOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All four GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start a GUARANTEED container when there's nothing left unallocated
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(4,
+ BuilderUtils.newResource(1024, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ add(createContainerId(3));
+ add(createContainerId(4));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0)) ||
+ status.getContainerId().equals(createContainerId(4))) {
+ Assert.assertEquals(
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ } else {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
+ * which in aggregate exceeds the capacity of the node. The first two
+ * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
+ * one utilizes nearly all of its resource requested. Then try to start two
+ * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
+ * expected to be queued immediately. Upon the completion of the
+ * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
+ * are expected to start.
+ */
+ @Test
+ public void testStartOppContainersUponContainerCompletion() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+
+ // All three containers are all expected to start immediately
+ // because the node utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the contianers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(4,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ // the two new OPPORTUNISTIC containers are expected to be queued
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(3), ContainerSubState.SCHEDULED);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+ allowContainerToSucceed(2);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.DONE);
+
+ // the two OPPORTUNISTIC containers are expected to start together
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED container that consumes all the resources on the
+ * node and keeps running, followed by two OPPORTUNISTIC containers that
+ * will be queued forever because there is no containers starting or
+ * finishing. Then try to start OPPORTUNISTIC containers out of band.
+ */
+ @Test
+ public void testStartOpportunisticContainersOutOfBand() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(0,
+ BuilderUtils.newResource(2048, 4), true))));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+
+ // the container is fully utilizing its resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+
+ // send two OPPORTUNISTIC container requests that are expected to be queued
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.SCHEDULED);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // the containers utilization dropped to the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start containers out of band.
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // no containers in queue are expected to be launched because the
+ // containers utilization is not below the over-allocation threshold
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.SCHEDULED);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+
+ // the containers utilization dropped way below the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // the two OPPORTUNISTIC containers are expected to be launched
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ private void setContainerResourceUtilization(ResourceUtilization usage) {
+ ((ContainerMonitorForOverallocationTest)
+ containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
+ }
+
+ private void allowContainerToSucceed(int containerId) {
+ ((LongRunningContainerSimulatingContainerExecutor) this.exec)
+ .containerSucceeded(createContainerId(containerId));
+ }
+
+
+ protected StartContainerRequest createStartContainerRequest(int containerId,
+ Resource resource, boolean isGuaranteed) throws IOException {
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
+ ExecutionType.OPPORTUNISTIC;
+ Token containerToken = createContainerToken(
+ createContainerId(containerId),
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
+ context.getContainerTokenSecretManager(),
+ null, executionType);
+
+ return StartContainerRequest.newInstance(
+ containerLaunchContext, containerToken);
+ }
+
+ protected void verifyContainerStatuses(
+ Map<ContainerId, ContainerSubState> expected)
+ throws IOException, YarnException {
+ List<ContainerId> statList = new ArrayList<>(expected.keySet());
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+
+ for (ContainerStatus status : containerStatuses) {
+ ContainerId containerId = status.getContainerId();
+ Assert.assertEquals(containerId + " is in unexpected state",
+ expected.get(containerId), status.getContainerSubState());
+ }
+ }
+
+ /**
+ * A container manager that sends a dummy container pid while it's cleaning
+ * up running containers. Used along with
+ * LongRunningContainerSimulatingContainerExecutor to simulate long running
+ * container processes for testing purposes.
+ */
+ private static class LongRunningContainerSimulatingContainersManager
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ LongRunningContainerSimulatingContainersManager(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ /**
+ * Create a container launcher that signals container processes
+ * with a dummy pid. The container processes are simulated in
+ * LongRunningContainerSimulatingContainerExecutor which does
+ * not write a pid file on behalf of containers to launch, so
+ * the pid does not matter.
+ */
+ @Override
+ protected ContainersLauncher createContainersLauncher(
+ Context context, ContainerExecutor exec) {
+ ContainerManagerImpl containerManager = this;
+ return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
+ this) {
+ @Override
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager) {
+ @Override
+ protected String getContainerPid(Path pidFilePath)
+ throws Exception {
+ return "123";
+ }
+
+ };
+ }
+ };
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForOverallocationTest(exec,
+ dispatcher, context);
+ }
+
+ public void startContainersOutOfBandUponLowUtilization() {
+ ((ContainerMonitorForOverallocationTest) getContainersMonitor())
+ .attemptToStartContainersUponLowUtilization();
+ }
+ }
+
+ /**
+ * A container executor that simulates long running container processes
+ * by having container launch threads sleep infinitely until it's given
+ * a signal to finish with either a success or failure exit code.
+ */
+ private static class LongRunningContainerSimulatingContainerExecutor
+ extends DefaultContainerExecutor {
+ private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
+ new ConcurrentHashMap<>();
+
+ public void containerSucceeded(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toSucceed();
+ }
+ }
+
+ public void containerFailed(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toFail();
+ }
+ }
+
+ /**
+ * Simulate long running container processes by having container launcher
+ * threads wait infinitely for a signal to finish.
+ */
+ @Override
+ public int launchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ ContainerId container = ctx.getContainer().getContainerId();
+ containers.putIfAbsent(container, new ContainerFinishLatch(container));
+
+ // simulate a long running container process by having the
+ // container launch thread sleep forever until it's given a
+ // signal to finish with a exit code.
+ while (!containers.get(container).toProceed) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ return -1;
+ }
+ }
+
+ return containers.get(container).getContainerExitCode();
+ }
+
+ /**
+ * Override signalContainer() so that simulated container processes
+ * are properly cleaned up.
+ */
+ @Override
+ public boolean signalContainer(ContainerSignalContext ctx)
+ throws IOException {
+ containerSucceeded(ctx.getContainer().getContainerId());
+ return true;
+ }
+
+ /**
+ * A signal that container launch threads wait for before exiting
+ * in order to simulate long running container processes.
+ */
+ private static final class ContainerFinishLatch {
+ volatile boolean toProceed;
+ int exitCode;
+ ContainerId container;
+
+ ContainerFinishLatch(ContainerId containerId) {
+ exitCode = 0;
+ toProceed = false;
+ container = containerId;
+ }
+
+ void toSucceed() {
+ exitCode = 0;
+ toProceed = true;
+ }
+
+ void toFail() {
+ exitCode = -101;
+ toProceed = true;
+ }
+
+ int getContainerExitCode() {
+ // read barrier of toProceed to make sure the exit code is not stale
+ if (toProceed) {
+ LOG.debug(container + " finished with exit code: " + exitCode);
+ }
+ return exitCode;
+ }
+ }
+ }
+
+ /**
+ * A test implementation of container monitor that allows control of
+ * current resource utilization.
+ */
+ private static class ContainerMonitorForOverallocationTest
+ extends ContainersMonitorImpl {
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForOverallocationTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, System.currentTimeMillis());
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/24] hadoop git commit: YARN-6739. Crash NM at start time if
oversubscription is on but LinuxContainerExcutor or cgroup is off.
Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6739. Crash NM at start time if oversubscription is on but LinuxContainerExcutor or cgroup is off. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0456591f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0456591f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0456591f
Branch: refs/heads/YARN-1011
Commit: 0456591fc1e13b9510720642b390c1d740f8f517
Parents: c045be5
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Apr 2 15:09:52 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:14:57 2018 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 25 ++++++++++++++++++++
1 file changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0456591f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 70a8ebf..7873882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -221,6 +222,7 @@ public class ContainersMonitorImpl extends AbstractService implements
initializeOverAllocation(conf);
if (context.isOverAllocationEnabled()) {
+ checkOverAllocationPrerequisites();
pmemCheckEnabled = true;
LOG.info("Force enabling physical memory checks because " +
"overallocation is enabled");
@@ -260,6 +262,29 @@ public class ContainersMonitorImpl extends AbstractService implements
super.serviceInit(this.conf);
}
+ /**
+ * Check all prerequisites for NM over-allocation.
+ */
+ private void checkOverAllocationPrerequisites() throws YarnException {
+ // LinuxContainerExecutor is required to enable overallocation
+ if (!(containerExecutor instanceof LinuxContainerExecutor)) {
+ throw new YarnException(LinuxContainerExecutor.class.getName() +
+ " is required for overallocation");
+ }
+ if (ResourceHandlerModule.getCGroupsHandler() == null) {
+ throw new YarnException("CGroups must be enabled to support" +
+ " overallocation");
+ }
+ if (ResourceHandlerModule.getCpuResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups cpu isolation must be enabled to support overallocation");
+ }
+ if (ResourceHandlerModule.getMemoryResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups memory isolation must be enabled for overallocation");
+ }
+ }
+
private boolean isContainerMonitorEnabled() {
return conf.getBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED,
YarnConfiguration.DEFAULT_NM_CONTAINER_MONITOR_ENABLED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/24] hadoop git commit: YARN-6794. Fair Scheduler to explicitly
promote OPPORTUNISITIC containers locally at the node where they're running.
Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6794. Fair Scheduler to explicitly promote OPPORTUNISITIC containers locally at the node where they're running. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1dae721
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1dae721
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1dae721
Branch: refs/heads/YARN-1011
Commit: c1dae721e8978fcfb35fb89d6e86f5b14df29bfd
Parents: f9db036
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jun 20 10:53:20 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:01 2018 -0700
----------------------------------------------------------------------
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../scheduler/SchedulerNode.java | 27 +
.../scheduler/fair/FSAppAttempt.java | 32 +-
.../scheduler/fair/FSSchedulerNode.java | 57 +-
.../scheduler/fair/FairScheduler.java | 47 +-
.../TestResourceTrackerService.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 569 +++++++++++++++++++
7 files changed, 715 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 25a9415..d812e57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -672,7 +672,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
public Resource getCurrentConsumption() {
- return attemptResourceUsage.getUsed();
+ return Resources.add(attemptResourceUsage.getUsed(),
+ attemptOpportunisticResourceUsage.getUsed());
}
private Container updateContainerAndNMToken(RMContainer rmContainer,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 3e16dc2..9e38d52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -282,6 +282,33 @@ public abstract class SchedulerNode {
return true;
}
+ /**
+ * Attempt to promote an OPPORTUNISTIC container that has been allocated.
+ * @param rmContainer the OPPORTUNISTIC container to promote
+ * @return true if the given OPPORTUNISTIC container is promoted,
+ * false otherwise
+ */
+ public synchronized boolean tryToPromoteOpportunisticContainer(
+ RMContainer rmContainer) {
+ assert (rmContainer.getExecutionType() == ExecutionType.OPPORTUNISTIC);
+
+ boolean promoted = false;
+ Resource resource = rmContainer.getContainer().getResource();
+ if (allocatedContainers.containsKey(rmContainer.getContainerId()) &&
+ Resources.fitsIn(resource, getUnallocatedResource())) {
+ Resources.subtractFrom(allocatedResourceOpportunistic, resource);
+ numOpportunisticContainers--;
+
+ Resources.addTo(allocatedResourceGuaranteed, resource);
+ numGuaranteedContainers++;
+ Resources.subtractFrom(unallocatedResource, resource);
+
+ promoted = true;
+ }
+
+ return promoted;
+ }
+
/**
* Get resources that are not allocated to GUARANTEED containers on the node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 7b32a9d..b68c245 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -505,6 +505,29 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
/**
+ * Update resource accounting upon promotion of an OPPORTUNISTIC container.
+ * @param rmContainer the OPPORTUNISTIC container that has been promoted
+ */
+ public void opportunisticContainerPromoted(RMContainer rmContainer) {
+ // only an OPPORTUNISTIC container can be promoted
+ assert (ExecutionType.OPPORTUNISTIC == rmContainer.getExecutionType());
+
+ // the container to be promoted must belong to the current app attempt
+ if (rmContainer.getApplicationAttemptId().equals(
+ getApplicationAttemptId())) {
+ Resource resource = rmContainer.getContainer().getResource();
+ try {
+ writeLock.lock();
+ attemptOpportunisticResourceUsage.decUsed(resource);
+ attemptResourceUsage.incUsed(resource);
+ getQueue().incUsedGuaranteedResource(resource);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ /**
* Should be called when the scheduler assigns a container at a higher
* degree of locality than the current threshold. Reset the allowed locality
* level to a higher degree of locality.
@@ -1159,7 +1182,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
*
* @param node
* Node that the application has an existing reservation on
- * @return whether the reservation on the given node is valid.
+ * @return true if the reservation is turned into an allocation
*/
boolean assignReservedContainer(FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
@@ -1186,8 +1209,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(),
node.getUnallocatedResource())) {
assignContainer(node, false, true);
+ return true;
}
- return true;
+ return false;
}
/**
@@ -1355,12 +1379,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override
public Resource getGuaranteedResourceUsage() {
- return getCurrentConsumption();
+ return Resources.clone(attemptResourceUsage.getUsed());
}
@Override
public Resource getOpportunisticResourceUsage() {
- return attemptOpportunisticResourceUsage.getUsed();
+ return Resources.clone(attemptOpportunisticResourceUsage.getUsed());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index a53dda4..efbe615 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -35,10 +36,13 @@ import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.util.resource.Resources;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;
@@ -66,6 +70,13 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ // The set of containers that need to be handled before resource
+ // available on the node can be assigned to resource requests.
+ // This is a queue of reserved and opportunistic containers on
+ // the node.
+ private final LinkedHashSet<RMContainer> priorityContainers =
+ new LinkedHashSet(1);
+
@VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
@@ -124,6 +135,7 @@ public class FSSchedulerNode extends SchedulerNode {
+ application.getApplicationId());
}
setReservedContainer(container);
+ priorityContainers.add(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
}
@@ -142,7 +154,7 @@ public class FSSchedulerNode extends SchedulerNode {
" for application " + reservedApplication.getApplicationId() +
" on node " + this);
}
-
+ priorityContainers.remove(getReservedContainer());
setReservedContainer(null);
this.reservedAppSchedulable = null;
}
@@ -274,6 +286,13 @@ public class FSSchedulerNode extends SchedulerNode {
} else {
LOG.error("Allocated empty container" + rmContainer.getContainerId());
}
+
+ // keep track of opportunistic containers allocated so that we can promote
+ // them before we assign resources available to resource requests.
+ if (ExecutionType.OPPORTUNISTIC.equals(
+ rmContainer.getContainer().getExecutionType())) {
+ priorityContainers.add(rmContainer);
+ }
}
/**
@@ -292,4 +311,40 @@ public class FSSchedulerNode extends SchedulerNode {
containersForPreemption.remove(container);
}
}
+
+ /**
+ * Try to assign resources available to reserved container and opportunistic
+ * containers that have been allocated.
+ * @return the list of opportunistic containers that have been promoted
+ */
+ public synchronized List<RMContainer> handlePriorityContainers() {
+ boolean assigned = true;
+ List<RMContainer> promotedContainers = new ArrayList<>(0);
+
+ List<RMContainer> candidateContainers = new ArrayList<>(priorityContainers);
+ for (RMContainer rmContainer : candidateContainers) {
+ boolean isReservedContainer =
+ rmContainer.getReservedSchedulerKey() != null;
+ if (isReservedContainer) {
+ // attempt to assign resources that have been reserved
+ FSAppAttempt reservedApp = getReservedAppSchedulable();
+ if (reservedApp != null) {
+ reservedApp.assignReservedContainer(this);
+ }
+ } else {
+ if (super.tryToPromoteOpportunisticContainer(rmContainer)) {
+ priorityContainers.remove(rmContainer);
+ assigned = true;
+ promotedContainers.add(rmContainer);
+ }
+ }
+
+ if (!assigned) {
+ // break out of the loop because assigned being false indicates
+ // there is no more resources that are available for promotion.
+ break;
+ }
+ }
+ return promotedContainers;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 25782a1..31f1961 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -70,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
@@ -104,6 +106,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -1113,8 +1116,8 @@ public class FairScheduler extends
// Assign new containers...
// 1. Ensure containers are assigned to the apps that preempted
- // 2. Check for reserved applications
- // 3. Schedule GUARANTEED containers if there are no reservations
+ // 2. Check for reserved applications or promote OPPORTUNISTIC containers
+ // 3. Schedule GUARANTEED containers
// 4. Schedule OPPORTUNISTIC containers if possible
// Apps may wait for preempted containers
@@ -1123,12 +1126,14 @@ public class FairScheduler extends
// when C does not qualify for preemption itself.
attemptToAssignPreemptedResources(node);
- boolean validReservation = attemptToAssignReservedResources(node);
- if (!validReservation) {
- // only attempt to assign GUARANTEED containers if there is no
- // reservation on the node because
- attemptToAssignResourcesAsGuaranteedContainers(node);
- }
+ // before we assign resources to outstanding resource requests, we
+ // need to assign the resources to either the container that has
+ // made a reservation or allocated OPPORTUNISTIC containers so that
+ // they can be promoted. This ensures that request requests that
+ // are eligible for guaranteed resources are satisfied in FIFO order
+ attemptToAssignReservedResourcesOrPromoteOpportunisticContainers(node);
+
+ attemptToAssignResourcesAsGuaranteedContainers(node);
// attempt to assign OPPORTUNISTIC containers regardless of whether
// we have made a reservation or assigned a GUARANTEED container
@@ -1143,15 +1148,27 @@ public class FairScheduler extends
}
/**
- * Assign the reserved resource to the application that have reserved it.
+ * Attempt to assign reserved resources and promote OPPORTUNISTIC containers
+ * thata have already been allocated.
*/
- private boolean attemptToAssignReservedResources(FSSchedulerNode node) {
- boolean success = false;
- FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
- if (reservedAppSchedulable != null) {
- success = reservedAppSchedulable.assignReservedContainer(node);
+ private void attemptToAssignReservedResourcesOrPromoteOpportunisticContainers(
+ FSSchedulerNode node) {
+ Map<Container, ContainerUpdateType> promotion = new HashMap<>(0);
+
+ List<RMContainer> promoted = node.handlePriorityContainers();
+ for (RMContainer rmContainer : promoted) {
+ FSAppAttempt appAttempt = getSchedulerApp(
+ rmContainer.getApplicationAttemptId());
+ appAttempt.opportunisticContainerPromoted(rmContainer);
+
+ promotion.put(rmContainer.getContainer(),
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE);
+ }
+
+ if (!promotion.isEmpty()) {
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeUpdateContainerEvent(node.getNodeID(), promotion));
}
- return success;
}
private void attemptToAssignResourcesAsGuaranteedContainers(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index e40b3c0..360d1af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2309,9 +2309,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rc.getExecutionType());
}
- // Should only include GUARANTEED resources
currentConsumption = applicationAttempt.getCurrentConsumption();
- Assert.assertEquals(Resource.newInstance(2048, 1), currentConsumption);
+ Assert.assertEquals(Resource.newInstance(5120, 3), currentConsumption);
allocResources =
applicationAttempt.getQueue().getMetrics().getAllocatedResources();
Assert.assertEquals(Resource.newInstance(2048, 1), allocResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1dae721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5878ccd..7fbf84a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3353,6 +3353,575 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test promotion of a single OPPORTUNISTIC container when no resources are
+ * reserved on the node where the container is allocated.
+ */
+ @Test
+ public void testSingleOpportunisticContainerPromotionWithoutReservation()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of two OPPORTUNISTIC containers when no resources are
+ * reserved on the node where the container is allocated.
+ */
+ @Test
+ public void testMultipleOpportunisticContainerPromotionWithoutReservation()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1536, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1536, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // node utilization is low after the third container run on the node
+ ContainerStatus container3Status = ContainerStatus.newInstance(
+ allocatedContainers3.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(container3Status),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(2000, 0, 0.2f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers4.get(0).getExecutionType());
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ // the first OPPORTUNISTIC container should be promoted
+ assertEquals(1536, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ // the second OPPORLTUNISTIC container should not be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ // the second OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of OPPORTUNISTIC container when there is resources
+ * reserved before the container is allocated. The scheduler should
+ * satisfy the reservation first before it promotes the OPPORTUNISTIC
+ * container when resources are released.
+ */
+ @Test
+ public void testOpportunisticContainerPromotionWithPriorReservation()
+ throws Exception {
+
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(2000, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers4.get(0).getExecutionType());
+ assertTrue("A reservation should still be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the reserved container of the third request that opted out of
+ // oversubscription should now be satisfied with a GUARANTEED container
+ assertEquals(2000, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("The reservation for the third request should be canceled",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+ // the OPPORTUNISTIC container should not be promoted given the released
+ // resources are taken by handling the reservation
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
+ /**
+ * Test promotion of OPPORTUNISTIC container when there is resources
+ * reserved after the container is allocated. The scheduler should
+ * promotes the OPPORTUNISTIC container before it satisfy the reservation
+ * when resources are released.
+ */
+ @Test
+ public void testOpportunisticContainerPromotionWithPostReservation()
+ throws Exception {
+
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create two scheduling requests that leave no unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(2048, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers2.get(0).getExecutionType());
+
+ // node utilization is low after the two container run on the node
+ ContainerStatus container1Status = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ ContainerStatus container2Status = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ List<ContainerStatus> containerStatuses = new ArrayList<>(2);
+ containerStatuses.add(container1Status);
+ containerStatuses.add(container2Status);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(containerStatuses, Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that asks for more than what's left
+ // unallocated on the node but can be served with overallocation.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers3.get(0).getExecutionType());
+ assertTrue("No reservation should be made for the third request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ // create another scheduling request that opts out of oversubscription
+ ApplicationAttemptId appAttempt4 =
+ createSchedulingRequest(2000, "queue3", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 0);
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the fourth request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the first GUARANTEED container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers1.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the OPPORTUNISTIC container should be promoted
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+ assertTrue("A reservation should still be made for the fourth request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(2000, 1)));
+
+ // now the second GUARANTEED container finishes
+ finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers2.get(0).getId(),
+ ContainerState.RUNNING, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(3000, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ // the reserved container of the fourth request that opted out of
+ // oversubscription should now be satisfied with a GUARANTEED container
+ assertEquals(2000, scheduler.getQueueManager().getQueue("queue3").
+ getGuaranteedResourceUsage().getMemorySize());
+ allocatedContainers4 =
+ scheduler.getSchedulerApp(appAttempt4).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers4.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers4.get(0).getExecutionType());
+ assertTrue("The reservation for the fourth request should be canceled",
+ scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/24] hadoop git commit: Revert "YARN-6675. Add NM support to
launch opportunistic containers based on overallocation. Contributed by Haibo
Chen."
Posted by ha...@apache.org.
Revert "YARN-6675. Add NM support to launch opportunistic containers based on overallocation. Contributed by Haibo Chen."
This reverts commit c1362b68af03c546f4c0802758e9729d2372ab6c.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8851333
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8851333
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8851333
Branch: refs/heads/YARN-1011
Commit: f8851333e8bf5e7d25b12839c5493c48de4046f8
Parents: 879a5e7
Author: Haibo Chen <ha...@apache.org>
Authored: Tue May 8 13:50:06 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:14:58 2018 -0700
----------------------------------------------------------------------
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 8 +-
.../launcher/ContainerLaunch.java | 2 +-
.../launcher/ContainersLauncher.java | 9 +-
.../monitor/ContainersMonitor.java | 38 +-
.../monitor/ContainersMonitorImpl.java | 56 +-
.../AllocationBasedResourceTracker.java | 114 --
...locationBasedResourceUtilizationTracker.java | 158 +++
.../scheduler/ContainerScheduler.java | 317 ++---
.../scheduler/ContainerSchedulerEventType.java | 4 +-
.../scheduler/NMAllocationPolicy.java | 63 -
.../scheduler/ResourceUtilizationTracker.java | 17 +-
.../SnapshotBasedOverAllocationPolicy.java | 54 -
.../UtilizationBasedResourceTracker.java | 95 --
.../BaseContainerManagerTest.java | 35 -
.../TestContainersMonitorResourceChange.java | 9 +-
.../TestAllocationBasedResourceTracker.java | 82 --
...locationBasedResourceUtilizationTracker.java | 93 ++
.../TestContainerSchedulerRecovery.java | 58 +-
...estContainerSchedulerWithOverAllocation.java | 1121 ------------------
20 files changed, 419 insertions(+), 1916 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index d757376..572684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -543,7 +543,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private ResourceUtilization getContainersUtilization() {
ContainersMonitor containersMonitor =
this.context.getContainerManager().getContainersMonitor();
- return containersMonitor.getContainersUtilization(false).getUtilization();
+ return containersMonitor.getContainersUtilization();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a08e227..27a7c80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -245,12 +245,6 @@ public class ContainerManagerImpl extends CompositeService implements
metrics);
addService(rsrcLocalizationSrvc);
- this.containersMonitor = createContainersMonitor(exec);
- addService(this.containersMonitor);
-
- // ContainersLauncher must be added after ContainersMonitor
- // because the former depends on the latter to initialize
- // over-allocation first.
containersLauncher = createContainersLauncher(context, exec);
addService(containersLauncher);
@@ -275,6 +269,8 @@ public class ContainerManagerImpl extends CompositeService implements
nmMetricsPublisher = createNMTimelinePublisher(context);
context.setNMTimelinePublisher(nmMetricsPublisher);
}
+ this.containersMonitor = createContainersMonitor(exec);
+ addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
new ContainerEventDispatcher());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 0228332..6347d4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1079,7 +1079,7 @@ public class ContainerLaunch implements Callable<Integer> {
* @return Process ID
* @throws Exception
*/
- protected String getContainerPid(Path pidFilePath) throws Exception {
+ private String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
container.getContainerId().toString();
String processId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 2f5acfa..7870f86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -121,7 +121,8 @@ public class ContainersLauncher extends AbstractService
containerId.getApplicationAttemptId().getApplicationId());
ContainerLaunch launch =
- createContainerLaunch(app, event.getContainer());
+ new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
+ event.getContainer(), dirsHandler, containerManager);
containerLauncher.submit(launch);
running.put(containerId, launch);
break;
@@ -224,10 +225,4 @@ public class ContainersLauncher extends AbstractService
break;
}
}
-
- protected ContainerLaunch createContainerLaunch(
- Application app, Container container) {
- return new ContainerLaunch(context, getConfig(), dispatcher,
- exec, app, container, dirsHandler, containerManager);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index 8da4ec4..64831e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -23,24 +23,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
public interface ContainersMonitor extends Service,
EventHandler<ContainersMonitorEvent>, ResourceView {
-
- /**
- * Get the aggregate resource utilization of containers running on the node,
- * with a timestamp of the measurement.
- * @param latest true if the latest result should be returned
- * @return ResourceUtilization resource utilization of all containers
- */
- ContainersResourceUtilization getContainersUtilization(boolean latest);
-
- /**
- * Get the policy to over-allocate containers when over-allocation is on.
- * @return null if over-allocation is turned off
- */
- NMAllocationPolicy getContainerOverAllocationPolicy();
+ ResourceUtilization getContainersUtilization();
float getVmemRatio();
@@ -80,26 +66,4 @@ public interface ContainersMonitor extends Service,
* containersMonitor.getVmemRatio());
resourceUtil.subtractFrom((int)resource.getMemorySize(), vmem, vCores);
}
-
- /**
- * A snapshot of resource utilization of all containers with the timestamp.
- */
- final class ContainersResourceUtilization {
- private final ResourceUtilization utilization;
- private final long timestamp;
-
- public ContainersResourceUtilization(
- ResourceUtilization utilization, long timestamp) {
- this.utilization = utilization;
- this.timestamp = timestamp;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ResourceUtilization getUtilization() {
- return utilization;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a045d78..7873882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -25,10 +25,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.NMAllocationPolicy;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.SnapshotBasedOverAllocationPolicy;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,9 +112,8 @@ public class ContainersMonitorImpl extends AbstractService implements
CPU, MEMORY
}
- private ContainersResourceUtilization latestContainersUtilization;
+ private ResourceUtilization containersUtilization;
- private NMAllocationPolicy overAllocationPolicy;
private ResourceThresholds overAllocationPreemptionThresholds;
private int overAlloctionPreemptionCpuCount = -1;
@@ -134,8 +129,7 @@ public class ContainersMonitorImpl extends AbstractService implements
this.monitoringThread = new MonitoringThread();
- this.latestContainersUtilization = new ContainersResourceUtilization(
- ResourceUtilization.newInstance(-1, -1, -1.0f), -1L);
+ this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
}
@Override
@@ -371,10 +365,6 @@ public class ContainersMonitorImpl extends AbstractService implements
this.overAllocationPreemptionThresholds = ResourceThresholds.newInstance(
cpuPreemptionThreshold, memoryPreemptionThreshold);
- // TODO make this configurable
- this.overAllocationPolicy =
- createOverAllocationPolicy(resourceThresholds);
-
LOG.info("NodeManager oversubscription enabled with overallocation " +
"thresholds (memory:" + overAllocationMemoryUtilizationThreshold +
", CPU:" + overAllocationCpuUtilizationThreshold + ") and preemption" +
@@ -382,11 +372,6 @@ public class ContainersMonitorImpl extends AbstractService implements
cpuPreemptionThreshold + ")");
}
- protected NMAllocationPolicy createOverAllocationPolicy(
- ResourceThresholds resourceThresholds) {
- return new SnapshotBasedOverAllocationPolicy(resourceThresholds, this);
- }
-
private boolean isResourceCalculatorAvailable() {
if (resourceCalculatorPlugin == null) {
LOG.info("ResourceCalculatorPlugin is unavailable on this system. " + this
@@ -670,12 +655,7 @@ public class ContainersMonitorImpl extends AbstractService implements
}
// Save the aggregated utilization of the containers
- setLatestContainersUtilization(trackedContainersUtilization);
-
- // check opportunity to start containers if over-allocation is on
- if (context.isOverAllocationEnabled()) {
- attemptToStartContainersUponLowUtilization();
- }
+ setContainersUtilization(trackedContainersUtilization);
// Publish the container utilization metrics to node manager
// metrics system.
@@ -1065,34 +1045,12 @@ public class ContainersMonitorImpl extends AbstractService implements
}
@Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- // TODO update containerUtilization if latest is true
- return this.latestContainersUtilization;
- }
-
- @Override
- public NMAllocationPolicy getContainerOverAllocationPolicy() {
- return overAllocationPolicy;
- }
-
- private void setLatestContainersUtilization(ResourceUtilization utilization) {
- this.latestContainersUtilization = new ContainersResourceUtilization(
- utilization, System.currentTimeMillis());
+ public ResourceUtilization getContainersUtilization() {
+ return this.containersUtilization;
}
- @VisibleForTesting
- public void attemptToStartContainersUponLowUtilization() {
- if (getContainerOverAllocationPolicy() != null) {
- Resource available = getContainerOverAllocationPolicy()
- .getAvailableResources();
- if (available.getMemorySize() > 0 &&
- available.getVirtualCores() > 0) {
- eventDispatcher.getEventHandler().handle(
- new ContainerSchedulerEvent(null,
- ContainerSchedulerEventType.SCHEDULE_CONTAINERS));
- }
- }
+ private void setContainersUtilization(ResourceUtilization utilization) {
+ this.containersUtilization = utilization;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
deleted file mode 100644
index 86b3698..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An implementation of the resource utilization tracker that equates
- * resource utilization with the total resource allocated to the container.
- */
-public class AllocationBasedResourceTracker
- implements ResourceUtilizationTracker {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
-
- private static final Resource UNAVAILABLE =
- Resource.newInstance(0, 0);
-
- private ResourceUtilization containersAllocation;
- private ContainerScheduler scheduler;
-
-
- AllocationBasedResourceTracker(ContainerScheduler scheduler) {
- this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
- this.scheduler = scheduler;
- }
-
- /**
- * Get the accumulation of totally allocated resources to containers.
- * @return ResourceUtilization Resource Utilization.
- */
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return this.containersAllocation;
- }
-
- /**
- * Get the amount of resources that have not been allocated to containers.
- * @return Resource resources that have not been allocated to containers.
- */
- protected Resource getUnallocatedResources() {
- // unallocated resources = node capacity - containers allocation
- // = -(container allocation - node capacity)
- ResourceUtilization allocationClone =
- ResourceUtilization.newInstance(containersAllocation);
- getContainersMonitor()
- .subtractNodeResourcesFromResourceUtilization(allocationClone);
-
- Resource unallocated = UNAVAILABLE;
- if (allocationClone.getCPU() <= 0 &&
- allocationClone.getPhysicalMemory() <= 0 &&
- allocationClone.getVirtualMemory() <= 0) {
- int cpu = Math.round(allocationClone.getCPU() *
- getContainersMonitor().getVCoresAllocatedForContainers());
- long memory = allocationClone.getPhysicalMemory();
- unallocated = Resource.newInstance(-memory, -cpu);
- }
- return unallocated;
- }
-
-
- @Override
- public Resource getAvailableResources() {
- return getUnallocatedResources();
- }
-
- /**
- * Add Container's resources to the accumulated allocation.
- * @param container Container.
- */
- @Override
- public void containerLaunched(Container container) {
- ContainersMonitor.increaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- /**
- * Subtract Container's resources to the accumulated allocation.
- * @param container Container.
- */
- @Override
- public void containerReleased(Container container) {
- ContainersMonitor.decreaseResourceUtilization(
- getContainersMonitor(), this.containersAllocation,
- container.getResource());
- }
-
- public ContainersMonitor getContainersMonitor() {
- return this.scheduler.getContainersMonitor();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
new file mode 100644
index 0000000..6e2b617
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation of the {@link ResourceUtilizationTracker} that equates
+ * resource utilization with the total resource allocated to the container.
+ */
+public class AllocationBasedResourceUtilizationTracker implements
+ ResourceUtilizationTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
+
+ private ResourceUtilization containersAllocation;
+ private ContainerScheduler scheduler;
+
+ AllocationBasedResourceUtilizationTracker(ContainerScheduler scheduler) {
+ this.containersAllocation = ResourceUtilization.newInstance(0, 0, 0.0f);
+ this.scheduler = scheduler;
+ }
+
+ /**
+ * Get the accumulation of totally allocated resources to a container.
+ * @return ResourceUtilization Resource Utilization.
+ */
+ @Override
+ public ResourceUtilization getCurrentUtilization() {
+ return this.containersAllocation;
+ }
+
+ /**
+ * Add Container's resources to the accumulated Utilization.
+ * @param container Container.
+ */
+ @Override
+ public void addContainerResources(Container container) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Subtract Container's resources to the accumulated Utilization.
+ * @param container Container.
+ */
+ @Override
+ public void subtractContainerResource(Container container) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), this.containersAllocation,
+ container.getResource());
+ }
+
+ /**
+ * Check if NM has resources available currently to run the container.
+ * @param container Container.
+ * @return True, if NM has resources available currently to run the container.
+ */
+ @Override
+ public boolean hasResourcesAvailable(Container container) {
+ long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
+ return hasResourcesAvailable(pMemBytes,
+ (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
+ container.getResource().getVirtualCores());
+ }
+
+ private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
+ int cpuVcores) {
+ // Check physical memory.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
+ this.containersAllocation.getPhysicalMemory(),
+ (pMemBytes >> 20),
+ (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
+ }
+ if (this.containersAllocation.getPhysicalMemory() +
+ (int) (pMemBytes >> 20) >
+ (int) (getContainersMonitor()
+ .getPmemAllocatedForContainers() >> 20)) {
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("before vMemCheck" +
+ "[isEnabled={}, current={} + asked={} > allowed={}]",
+ getContainersMonitor().isVmemCheckEnabled(),
+ this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
+ (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
+ }
+ // Check virtual memory.
+ if (getContainersMonitor().isVmemCheckEnabled() &&
+ this.containersAllocation.getVirtualMemory() +
+ (int) (vMemBytes >> 20) >
+ (int) (getContainersMonitor()
+ .getVmemAllocatedForContainers() >> 20)) {
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("before cpuCheck [asked={} > allowed={}]",
+ this.containersAllocation.getCPU(),
+ getContainersMonitor().getVCoresAllocatedForContainers());
+ }
+ // Check CPU. Compare using integral values of cores to avoid decimal
+ // inaccuracies.
+ if (!hasEnoughCpu(this.containersAllocation.getCPU(),
+ getContainersMonitor().getVCoresAllocatedForContainers(), cpuVcores)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns whether there is enough space for coresRequested in totalCores.
+ * Converts currentAllocation usage to nearest integer count before comparing,
+ * as floats are inherently imprecise. NOTE: this calculation assumes that
+ * requested core counts must be integers, and currentAllocation core count
+ * must also be an integer.
+ *
+ * @param currentAllocation The current allocation, a float value from 0 to 1.
+ * @param totalCores The total cores in the system.
+ * @param coresRequested The number of cores requested.
+ * @return True if currentAllocationtotalCores*coresRequested <=
+ * totalCores.
+ */
+ public boolean hasEnoughCpu(float currentAllocation, long totalCores,
+ int coresRequested) {
+ // Must not cast here, as it would truncate the decimal digits.
+ return Math.round(currentAllocation * totalCores)
+ + coresRequested <= totalCores;
+ }
+
+ public ContainersMonitor getContainersMonitor() {
+ return this.scheduler.getContainersMonitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0bebe44..a61b9d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -49,7 +48,6 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,10 +80,6 @@ public class ContainerScheduler extends AbstractService implements
// Queue of Guaranteed Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedGuaranteedContainers = new LinkedHashMap<>();
- // sum of the resources requested by guaranteed containers in queue
- private final Resource guaranteedResourcesDemanded =
- Resource.newInstance(0, 0);
-
// Queue of Opportunistic Containers waiting for resources to run
private final LinkedHashMap<ContainerId, Container>
queuedOpportunisticContainers = new LinkedHashMap<>();
@@ -94,10 +88,6 @@ public class ContainerScheduler extends AbstractService implements
// or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
- // sum of the resources to be released by opportunistic containers that
- // have been marked to be killed or paused.
- private final Resource opportunisticResourcesToBeReleased =
- Resource.newInstance(0, 0);
// Containers launched by the Scheduler will take a while to actually
// move to the RUNNING state, but should still be fair game for killing
@@ -138,17 +128,6 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
- @VisibleForTesting
- public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
- NodeManagerMetrics metrics, int qLength) {
- super(ContainerScheduler.class.getName());
- this.context = context;
- this.dispatcher = dispatcher;
- this.metrics = metrics;
- this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
- this.opportunisticContainersStatus =
- OpportunisticContainersStatus.newInstance();
- }
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -176,16 +155,20 @@ public class ContainerScheduler extends AbstractService implements
YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
YarnConfiguration.
DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
- // We assume over allocation configurations have been initialized
- this.utilizationTracker = getResourceTracker();
}
- private AllocationBasedResourceTracker getResourceTracker() {
- if (context.isOverAllocationEnabled()) {
- return new UtilizationBasedResourceTracker(this);
- } else {
- return new AllocationBasedResourceTracker(this);
- }
+ @VisibleForTesting
+ public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+ NodeManagerMetrics metrics, int qLength) {
+ super(ContainerScheduler.class.getName());
+ this.context = context;
+ this.dispatcher = dispatcher;
+ this.metrics = metrics;
+ this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
+ this.utilizationTracker =
+ new AllocationBasedResourceUtilizationTracker(this);
+ this.opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
}
/**
@@ -208,18 +191,14 @@ public class ContainerScheduler extends AbstractService implements
if (event instanceof UpdateContainerSchedulerEvent) {
onUpdateContainer((UpdateContainerSchedulerEvent) event);
} else {
- LOG.error("Unknown event type on UpdateContainer: " + event.getType());
+ LOG.error("Unknown event type on UpdateCOntainer: " + event.getType());
}
break;
case SHED_QUEUED_CONTAINERS:
shedQueuedOpportunisticContainers();
break;
case RECOVERY_COMPLETED:
- startPendingContainers(false);
- break;
- case SCHEDULE_CONTAINERS:
- startPendingContainers(true);
- break;
+ startPendingContainers(maxOppQueueLength <= 0);
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
+ event.toString());
@@ -234,10 +213,10 @@ public class ContainerScheduler extends AbstractService implements
ContainerId containerId = updateEvent.getContainer().getContainerId();
if (updateEvent.isResourceChange()) {
if (runningContainers.containsKey(containerId)) {
- this.utilizationTracker.containerReleased(
+ this.utilizationTracker.subtractContainerResource(
new ContainerImpl(getConfig(), null, null, null, null,
updateEvent.getOriginalToken(), context));
- this.utilizationTracker.containerLaunched(
+ this.utilizationTracker.addContainerResources(
updateEvent.getContainer());
getContainersMonitor().handle(
new ChangeMonitoringContainerResourceEvent(containerId,
@@ -253,20 +232,17 @@ public class ContainerScheduler extends AbstractService implements
if (queuedOpportunisticContainers.remove(containerId) != null) {
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
- Resources.addTo(guaranteedResourcesDemanded,
- updateEvent.getContainer().getResource());
- startPendingContainers(true);
+ //Kill/pause opportunistic containers if any to make room for
+ // promotion request
+ reclaimOpportunisticContainerResources(updateEvent.getContainer());
}
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
// containers
if (queuedGuaranteedContainers.remove(containerId) != null) {
- Resources.subtractFrom(guaranteedResourcesDemanded,
- updateEvent.getContainer().getResource());
queuedOpportunisticContainers.put(containerId,
updateEvent.getContainer());
- startPendingContainers(false);
}
}
try {
@@ -293,7 +269,6 @@ public class ContainerScheduler extends AbstractService implements
|| rcs.getStatus() == RecoveredContainerStatus.PAUSED) {
if (execType == ExecutionType.GUARANTEED) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
} else if (execType == ExecutionType.OPPORTUNISTIC) {
queuedOpportunisticContainers
.put(container.getContainerId(), container);
@@ -304,7 +279,7 @@ public class ContainerScheduler extends AbstractService implements
}
} else if (rcs.getStatus() == RecoveredContainerStatus.LAUNCHED) {
runningContainers.put(container.getContainerId(), container);
- utilizationTracker.containerLaunched(container);
+ utilizationTracker.addContainerResources(container);
}
if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED
&& rcs.getCapability() != null) {
@@ -369,107 +344,65 @@ public class ContainerScheduler extends AbstractService implements
}
private void onResourcesReclaimed(Container container) {
- ContainerId containerId = container.getContainerId();
-
- // This could be killed externally for eg. by the ContainerManager,
- // in which case, the container might still be queued.
- if (queuedOpportunisticContainers.remove(containerId) != null) {
- return;
- }
+ oppContainersToKill.remove(container.getContainerId());
// This could be killed externally for eg. by the ContainerManager,
// in which case, the container might still be queued.
- if (queuedGuaranteedContainers.remove(containerId) != null) {
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
- return;
- }
-
- if (oppContainersToKill.remove(containerId) != null) {
- Resources.subtractFrom(
- opportunisticResourcesToBeReleased, container.getResource());
+ Container queued =
+ queuedOpportunisticContainers.remove(container.getContainerId());
+ if (queued == null) {
+ queuedGuaranteedContainers.remove(container.getContainerId());
}
// Requeue PAUSED containers
if (container.getContainerState() == ContainerState.PAUSED) {
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.GUARANTEED) {
- queuedGuaranteedContainers.put(containerId, container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
+ queuedGuaranteedContainers.put(container.getContainerId(), container);
} else {
- queuedOpportunisticContainers.put(containerId, container);
+ queuedOpportunisticContainers.put(
+ container.getContainerId(), container);
}
}
// decrement only if it was a running container
- Container completedContainer = runningContainers.remove(containerId);
+ Container completedContainer = runningContainers.remove(container
+ .getContainerId());
// only a running container releases resources upon completion
boolean resourceReleased = completedContainer != null;
if (resourceReleased) {
- this.utilizationTracker.containerReleased(container);
+ this.utilizationTracker.subtractContainerResource(container);
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
this.metrics.completeOpportunisticContainer(container.getResource());
}
-
- // In case of over-allocation being turned on, we may need to reclaim
- // more resources since the opportunistic containers that have been
- // killed or paused may have not released as much resource as we need.
- boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
- startPendingContainers(reclaimOpportunisticResources);
+ boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+ startPendingContainers(forceStartGuaranteedContainers);
}
}
/**
* Start pending containers in the queue.
- * @param reclaimOpportunisticResources if set to true, resources allocated
- * to running OPPORTUNISTIC containers will be reclaimed in
- * cases where there are GUARANTEED containers being queued
+ * @param forceStartGuaranteedContaieners When this is true, start guaranteed
+ * container without looking at available resource
*/
- private void startPendingContainers(boolean reclaimOpportunisticResources) {
- // When opportunistic container not allowed (which is determined by
- // max-queue length of pending opportunistic containers <= 0), start
- // guaranteed containers without looking at available resources and
- // skip scanning the queue of opportunistic containers
- if (maxOppQueueLength <= 0) {
- forcefullyStartGuaranteedContainers();
- return;
- }
-
- Resource available = utilizationTracker.getAvailableResources();
-
- // Start guaranteed containers that are queued, if resources available.
- boolean allGuaranteedContainersLaunched =
- startGuaranteedContainers(available);
- // Start opportunistic containers, if resources available, which is true
- // if all guaranteed containers in queue have been launched.
- if (allGuaranteedContainersLaunched) {
- startOpportunisticContainers(available);
- } else {
- // If not all guaranteed containers in queue are launched, we may need
- // to reclaim resources from opportunistic containers that are running.
- if (reclaimOpportunisticResources) {
- reclaimOpportunisticContainerResources();
- }
+ private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
+ // Start guaranteed containers that are paused, if resources available.
+ boolean resourcesAvailable = startContainers(
+ queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
+ // Start opportunistic containers, if resources available.
+ if (resourcesAvailable) {
+ startContainers(queuedOpportunisticContainers.values(), false);
}
}
- /**
- * Try to launch as many GUARANTEED containers as possible.
- * @param available the amount of resources available to launch containers
- * @return true if all queued GUARANTEED containers are launched
- * or there is no GUARANTEED containers to launch
- */
- private boolean startGuaranteedContainers(Resource available) {
- Iterator<Container> cIter =
- queuedGuaranteedContainers.values().iterator();
+ private boolean startContainers(
+ Collection<Container> containersToBeStarted, boolean force) {
+ Iterator<Container> cIter = containersToBeStarted.iterator();
boolean resourcesAvailable = true;
while (cIter.hasNext() && resourcesAvailable) {
Container container = cIter.next();
- if (isResourceAvailable(available, container)) {
- startContainer(container);
- Resources.subtractFrom(available, container.getResource());
+ if (tryStartContainer(container, force)) {
cIter.remove();
- Resources.subtractFrom(
- guaranteedResourcesDemanded, container.getResource());
} else {
resourcesAvailable = false;
}
@@ -477,49 +410,25 @@ public class ContainerScheduler extends AbstractService implements
return resourcesAvailable;
}
- /**
- * Launch all queued GUARANTEED containers without checking resource
- * availability. This is an optimization in cases where OPPORTUNISTIC
- * containers are not allowed on the node.
- */
- private void forcefullyStartGuaranteedContainers() {
- Iterator<Container> cIter =
- queuedGuaranteedContainers.values().iterator();
- while (cIter.hasNext()) {
- Container container = cIter.next();
+ private boolean tryStartContainer(Container container, boolean force) {
+ boolean containerStarted = false;
+ // call startContainer without checking available resource when force==true
+ if (force || resourceAvailableToStartContainer(
+ container)) {
startContainer(container);
- cIter.remove();
- Resources.subtractFrom(
- guaranteedResourcesDemanded, container.getResource());
+ containerStarted = true;
}
+ return containerStarted;
}
+
/**
- * Try to launch as many OPPORTUNISTIC containers as possible.
- * @param available the amount of resources available to launch containers
- * @return true if all OPPORTUNISTIC containers are launched
- * or there is no OPPORTUNISTIC containers to launch
+ * Check if there is resource available to start a given container
+ * immediately. (This can be extended to include overallocated resources)
+ * @param container the container to start
+ * @return true if container can be launched directly
*/
- private boolean startOpportunisticContainers(Resource available) {
- Iterator<Container> cIter =
- queuedOpportunisticContainers.values().iterator();
- boolean resourcesAvailable = true;
- while (cIter.hasNext() && resourcesAvailable) {
- Container container = cIter.next();
- if (isResourceAvailable(available, container)) {
- startContainer(container);
- Resources.subtractFrom(available, container.getResource());
- cIter.remove();
- } else {
- resourcesAvailable = false;
- }
- }
- return resourcesAvailable;
- }
-
- private static boolean isResourceAvailable(
- Resource resource, Container container) {
- Resource left = Resources.subtract(resource, container.getResource());
- return left.getMemorySize() >= 0 && left.getVirtualCores() >= 0;
+ private boolean resourceAvailableToStartContainer(Container container) {
+ return this.utilizationTracker.hasResourcesAvailable(container);
}
private boolean enqueueContainer(Container container) {
@@ -529,7 +438,6 @@ public class ContainerScheduler extends AbstractService implements
boolean isQueued;
if (isGuaranteedContainer) {
queuedGuaranteedContainers.put(container.getContainerId(), container);
- Resources.addTo(guaranteedResourcesDemanded, container.getResource());
isQueued = true;
} else {
if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
@@ -574,7 +482,18 @@ public class ContainerScheduler extends AbstractService implements
// enough number of opportunistic containers.
if (isGuaranteedContainer) {
enqueueContainer(container);
- startPendingContainers(true);
+
+ // When opportunistic container not allowed (which is determined by
+ // max-queue length of pending opportunistic containers <= 0), start
+ // guaranteed containers without looking at available resources.
+ boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+ startPendingContainers(forceStartGuaranteedContainers);
+
+ // if the guaranteed container is queued, we need to preempt opportunistic
+ // containers for make room for it
+ if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
+ reclaimOpportunisticContainerResources(container);
+ }
} else {
// Given an opportunistic container, we first try to start as many queuing
// guaranteed containers as possible followed by queuing opportunistic
@@ -592,19 +511,19 @@ public class ContainerScheduler extends AbstractService implements
}
@SuppressWarnings("unchecked")
- private void reclaimOpportunisticContainerResources() {
+ private void reclaimOpportunisticContainerResources(Container container) {
List<Container> extraOppContainersToReclaim =
- pickOpportunisticContainersToReclaimResources();
- killOpportunisticContainers(extraOppContainersToReclaim);
- }
-
- private void killOpportunisticContainers(
- Collection<Container> containersToReclaim) {
- for (Container contToReclaim : containersToReclaim) {
+ pickOpportunisticContainersToReclaimResources(
+ container.getContainerId());
+ // Kill the opportunistic containers that were chosen.
+ for (Container contToReclaim : extraOppContainersToReclaim) {
String preemptionAction = usePauseEventForPreemption == true ? "paused" :
- "preempted";
- LOG.info("Container {} will be {} to start the execution of guaranteed" +
- " containers.", contToReclaim.getContainerId(), preemptionAction);
+ "resumed";
+ LOG.info(
+ "Container {} will be {} to start the "
+ + "execution of guaranteed container {}.",
+ contToReclaim.getContainerId(), preemptionAction,
+ container.getContainerId());
if (usePauseEventForPreemption) {
contToReclaim.sendPauseEvent(
@@ -615,8 +534,6 @@ public class ContainerScheduler extends AbstractService implements
"Container Killed to make room for Guaranteed Container.");
}
oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
- Resources.addTo(
- opportunisticResourcesToBeReleased, contToReclaim.getResource());
}
}
@@ -625,7 +542,7 @@ public class ContainerScheduler extends AbstractService implements
// Skip to put into runningContainers and addUtilization when recover
if (!runningContainers.containsKey(container.getContainerId())) {
runningContainers.put(container.getContainerId(), container);
- this.utilizationTracker.containerLaunched(container);
+ this.utilizationTracker.addContainerResources(container);
}
if (container.getContainerTokenIdentifier().getExecutionType() ==
ExecutionType.OPPORTUNISTIC) {
@@ -634,12 +551,14 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToReclaimResources() {
+ private List<Container> pickOpportunisticContainersToReclaimResources(
+ ContainerId containerToStartId) {
// The opportunistic containers that need to be killed for the
// given container to start.
List<Container> extraOpportContainersToKill = new ArrayList<>();
// Track resources that need to be freed.
- ResourceUtilization resourcesToFreeUp = resourcesToFreeUp();
+ ResourceUtilization resourcesToFreeUp = resourcesToFreeUp(
+ containerToStartId);
// Go over the running opportunistic containers.
// Use a descending iterator to kill more recently started containers.
@@ -658,19 +577,15 @@ public class ContainerScheduler extends AbstractService implements
continue;
}
extraOpportContainersToKill.add(runningCont);
- // In the case of over-allocation, the running container may not
- // release as much resources as it has requested, but we'll check
- // again if more containers need to be killed/paused when this
- // container is released.
ContainersMonitor.decreaseResourceUtilization(
getContainersMonitor(), resourcesToFreeUp,
runningCont.getResource());
}
}
if (!hasSufficientResources(resourcesToFreeUp)) {
- LOG.warn("There are no sufficient resources to start guaranteed" +
- " containers at the moment. Opportunistic containers are in" +
- " the process of being killed to make room.");
+ LOG.warn("There are no sufficient resources to start guaranteed [{}]" +
+ "at the moment. Opportunistic containers are in the process of" +
+ "being killed to make room.", containerToStartId);
}
return extraOpportContainersToKill;
}
@@ -685,42 +600,34 @@ public class ContainerScheduler extends AbstractService implements
* getContainersMonitor().getVCoresAllocatedForContainers()) <= 0;
}
- /**
- * Determine how much resources are needed to be freed up to launch the given
- * GUARANTEED container. Used to determine how many running OPPORTUNISTIC
- * containers need to be killed/paused, assuming OPPORTUNISTIC containers to
- * be killed/paused will release the amount of resources they have requested.
- *
- * If the node is over-allocating itself, this may cause not enough
- * OPPORTUNISTIC containers being killed/paused in cases where the running
- * OPPORTUNISTIC containers are not consuming fully their resource requests.
- * We'd check again upon container completion events to see if more running
- * OPPORTUNISTIC containers need to be killed/paused.
- *
- * @return the amount of resource needed to be reclaimed for this container
- */
- private ResourceUtilization resourcesToFreeUp() {
+ private ResourceUtilization resourcesToFreeUp(
+ ContainerId containerToStartId) {
// Get allocation of currently allocated containers.
ResourceUtilization resourceAllocationToFreeUp = ResourceUtilization
- .newInstance(0, 0, 0.0f);
-
- // Add to the allocation the allocation of pending guaranteed containers.
- ContainersMonitor.increaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, guaranteedResourcesDemanded);
+ .newInstance(this.utilizationTracker.getCurrentUtilization());
+
+ // Add to the allocation the allocation of the pending guaranteed
+ // containers that will start before the current container will be started.
+ for (Container container : queuedGuaranteedContainers.values()) {
+ ContainersMonitor.increaseResourceUtilization(
+ getContainersMonitor(), resourceAllocationToFreeUp,
+ container.getResource());
+ if (container.getContainerId().equals(containerToStartId)) {
+ break;
+ }
+ }
// These resources are being freed, likely at the behest of another
// guaranteed container..
- ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, opportunisticResourcesToBeReleased);
-
- // Deduct any remaining resources available
- Resource availableResources = utilizationTracker.getAvailableResources();
- if (availableResources.getVirtualCores() > 0 &&
- availableResources.getMemorySize() > 0) {
- ContainersMonitor.decreaseResourceUtilization(getContainersMonitor(),
- resourceAllocationToFreeUp, availableResources);
+ for (Container container : oppContainersToKill.values()) {
+ ContainersMonitor.decreaseResourceUtilization(
+ getContainersMonitor(), resourceAllocationToFreeUp,
+ container.getResource());
}
+ // Subtract the overall node resources.
+ getContainersMonitor().subtractNodeResourcesFromResourceUtilization(
+ resourceAllocationToFreeUp);
return resourceAllocationToFreeUp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 9ad4f91..294eddf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -28,7 +28,5 @@ public enum ContainerSchedulerEventType {
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
CONTAINER_PAUSED,
- RECOVERY_COMPLETED,
- // Producer: Containers Monitor when over-allocation is on
- SCHEDULE_CONTAINERS
+ RECOVERY_COMPLETED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
deleted file mode 100644
index 58b73d2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/NMAllocationPolicy.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-
-/**
- * Keeps track of containers utilization over time and determines how much
- * resources are available to launch containers when over-allocation is on.
- */
-public abstract class NMAllocationPolicy {
- protected final ResourceThresholds overAllocationThresholds;
- protected final ContainersMonitor containersMonitor;
-
- public NMAllocationPolicy(
- ResourceThresholds overAllocationThresholds,
- ContainersMonitor containersMonitor) {
- this.containersMonitor = containersMonitor;
- this.overAllocationThresholds = overAllocationThresholds;
- }
-
- /**
- * Handle container launch events.
- * @param container the container that has been launched
- */
- public void containerLaunched(Container container) {
-
- }
-
- /**
- * Handle container release events.
- * @param container the container that has been released
- */
- public void containerReleased(Container container) {
-
- }
-
- /**
- * Get the amount of resources to launch containers when
- * over-allocation is turned on.
- * @return the amount of resources available to launch containers
- */
- public abstract Resource getAvailableResources();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 98d99c6..3c17eca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -39,20 +38,22 @@ public interface ResourceUtilizationTracker {
ResourceUtilization getCurrentUtilization();
/**
- * Get the amount of resources currently available to launch containers.
- * @return Resource resources available to launch containers
+ * Add Container's resources to Node Utilization.
+ * @param container Container.
*/
- Resource getAvailableResources();
+ void addContainerResources(Container container);
/**
- * Add Container's resources to Node Utilization upon container launch.
+ * Subtract Container's resources to Node Utilization.
* @param container Container.
*/
- void containerLaunched(Container container);
+ void subtractContainerResource(Container container);
/**
- * Subtract Container's resources to Node Utilization upon container release.
+ * Check if NM has resources available currently to run the container.
* @param container Container.
+ * @return True, if NM has resources available currently to run the container.
*/
- void containerReleased(Container container);
+ boolean hasResourcesAvailable(Container container);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
deleted file mode 100644
index f486506..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/SnapshotBasedOverAllocationPolicy.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.api.records.ResourceThresholds;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-
-/**
- * An implementation of NMAllocationPolicy based on the
- * snapshot of the latest containers utilization to determine how much
- * resources are available * to launch containers when over-allocation
- * is turned on.
- */
-public class SnapshotBasedOverAllocationPolicy
- extends NMAllocationPolicy {
-
- public SnapshotBasedOverAllocationPolicy(
- ResourceThresholds overAllocationThresholds,
- ContainersMonitor containersMonitor) {
- super(overAllocationThresholds, containersMonitor);
- }
-
- @Override
- public Resource getAvailableResources() {
- ResourceUtilization utilization =
- containersMonitor.getContainersUtilization(true).getUtilization();
- long memoryAvailable = Math.round(
- overAllocationThresholds.getMemoryThreshold() *
- containersMonitor.getPmemAllocatedForContainers()) -
- (utilization.getPhysicalMemory() << 20);
- int vcoreAvailable = Math.round(
- (overAllocationThresholds.getCpuThreshold() - utilization.getCPU()) *
- containersMonitor.getVCoresAllocatedForContainers());
- return Resource.newInstance(memoryAvailable >> 20, vcoreAvailable);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
deleted file mode 100644
index 6f9bc82..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
-* An resource availability tracker that determines if there are resources
-* available based on if there are unallocated resources or if there are
-* un-utilized resources.
-*/
-public class UtilizationBasedResourceTracker
- extends AllocationBasedResourceTracker {
- private static final Logger LOG =
- LoggerFactory.getLogger(AllocationBasedResourceTracker.class);
-
- private final NMAllocationPolicy overAllocationPolicy;
-
- UtilizationBasedResourceTracker(ContainerScheduler scheduler) {
- super(scheduler);
- this.overAllocationPolicy =
- getContainersMonitor().getContainerOverAllocationPolicy();
- }
-
- @Override
- public void containerLaunched(Container container) {
- super.containerLaunched(container);
- if (overAllocationPolicy != null) {
- overAllocationPolicy.containerLaunched(container);
- }
- }
-
- @Override
- public void containerReleased(Container container) {
- super.containerReleased(container);
- if (overAllocationPolicy != null) {
- overAllocationPolicy.containerReleased(container);
- }
- }
-
- @Override
- public Resource getAvailableResources() {
- Resource resourceBasedOnAllocation = getUnallocatedResources();
- Resource resourceBasedOnUtilization =
- getResourcesAvailableBasedOnUtilization();
- if (LOG.isDebugEnabled()) {
- LOG.debug("The amount of resources available based on allocation is " +
- resourceBasedOnAllocation + ", based on utilization is " +
- resourceBasedOnUtilization);
- }
-
- return Resources.componentwiseMax(resourceBasedOnAllocation,
- resourceBasedOnUtilization);
- }
-
- /**
- * Get the amount of resources based on the slack between
- * the actual utilization and desired utilization.
- * @return Resource resource available
- */
- private Resource getResourcesAvailableBasedOnUtilization() {
- if (overAllocationPolicy == null) {
- return Resources.none();
- }
-
- return overAllocationPolicy.getAvailableResources();
- }
-
- @Override
- public ResourceUtilization getCurrentUtilization() {
- return getContainersMonitor().getContainersUtilization(false)
- .getUtilization();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 92613ed..493aa4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -26,7 +26,6 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyString;
-import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -346,40 +345,6 @@ public abstract class BaseContainerManagerTest {
fStates.contains(containerStatus.getState()));
}
- public static void waitForContainerSubState(
- ContainerManagementProtocol containerManager, ContainerId containerID,
- ContainerSubState finalState)
- throws InterruptedException, YarnException, IOException {
- waitForContainerSubState(containerManager, containerID,
- Arrays.asList(finalState), 20);
- }
- public static void waitForContainerSubState(
- ContainerManagementProtocol containerManager, ContainerId containerID,
- List<ContainerSubState> finalStates, int timeOutMax)
- throws InterruptedException, YarnException, IOException {
- List<ContainerId> list = new ArrayList<>();
- list.add(containerID);
- GetContainerStatusesRequest request =
- GetContainerStatusesRequest.newInstance(list);
- ContainerStatus containerStatus;
- HashSet<ContainerSubState> fStates = new HashSet<>(finalStates);
- int timeoutSecs = 0;
- do {
- Thread.sleep(1000);
- containerStatus =
- containerManager.getContainerStatuses(request)
- .getContainerStatuses().get(0);
- LOG.info("Waiting for container to get into one of states " + fStates
- + ". Current state is " + containerStatus.getContainerSubState());
- timeoutSecs += 1;
- } while (!fStates.contains(containerStatus.getContainerSubState())
- && timeoutSecs < timeOutMax);
- LOG.info("Container state is " + containerStatus.getContainerSubState());
- Assert.assertTrue("ContainerSubState is not correct (timedout)",
- fStates.contains(containerStatus.getContainerSubState()));
- }
-
-
public static void waitForApplicationState(
ContainerManagerImpl containerManager, ApplicationId appID,
ApplicationState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index c071283..8aee532 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -288,7 +288,7 @@ public class TestContainersMonitorResourceChange {
// will be 0.
assertEquals(
"Resource utilization must be default with MonitorThread's first run",
- 0, containersMonitor.getContainersUtilization(false).getUtilization()
+ 0, containersMonitor.getContainersUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
// Verify the container utilization value. Since atleast one round is done,
@@ -303,9 +303,8 @@ public class TestContainersMonitorResourceChange {
ContainersMonitorImpl containersMonitor, int timeoutMsecs)
throws InterruptedException {
int timeWaiting = 0;
- while (0 == containersMonitor.getContainersUtilization(false)
- .getUtilization().compareTo(
- ResourceUtilization.newInstance(0, 0, 0.0f))) {
+ while (0 == containersMonitor.getContainersUtilization()
+ .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))) {
if (timeWaiting >= timeoutMsecs) {
break;
}
@@ -317,7 +316,7 @@ public class TestContainersMonitorResourceChange {
}
assertTrue("Resource utilization is not changed from second run onwards",
- 0 != containersMonitor.getContainersUtilization(false).getUtilization()
+ 0 != containersMonitor.getContainersUtilization()
.compareTo(ResourceUtilization.newInstance(0, 0, 0.0f)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
deleted file mode 100644
index 1e8bfdf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceTracker.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Tests for the {@link AllocationBasedResourceTracker} class.
- */
-public class TestAllocationBasedResourceTracker {
-
- private ContainerScheduler mockContainerScheduler;
-
- @Before
- public void setup() {
- mockContainerScheduler = mock(ContainerScheduler.class);
- ContainersMonitor containersMonitor =
- new ContainersMonitorImpl(mock(ContainerExecutor.class),
- mock(AsyncDispatcher.class), mock(Context.class));
- YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
- conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
- conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
- conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
- conf.setInt(YarnConfiguration.NM_VCORES, 8);
- containersMonitor.init(conf);
- when(mockContainerScheduler.getContainersMonitor())
- .thenReturn(containersMonitor);
- }
-
- /**
- * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
- * hasResourceAvailable should return false.
- */
- @Test
- public void testHasResourcesAvailable() {
- AllocationBasedResourceTracker tracker =
- new AllocationBasedResourceTracker(mockContainerScheduler);
- Container testContainer = mock(Container.class);
- when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
- for (int i = 0; i < 2; i++) {
- Assert.assertTrue(
- isResourcesAvailable(tracker.getAvailableResources(), testContainer));
- tracker.containerLaunched(testContainer);
- }
- Assert.assertFalse(
- isResourcesAvailable(tracker.getAvailableResources(), testContainer));
- }
-
- private static boolean isResourcesAvailable(
- Resource available, Container container) {
- return available.compareTo(container.getResource()) >= 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
new file mode 100644
index 0000000..82c2147
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestAllocationBasedResourceUtilizationTracker.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link AllocationBasedResourceUtilizationTracker} class.
+ */
+public class TestAllocationBasedResourceUtilizationTracker {
+
+ private ContainerScheduler mockContainerScheduler;
+
+ @Before
+ public void setup() {
+ mockContainerScheduler = mock(ContainerScheduler.class);
+ ContainersMonitor containersMonitor =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, 2.0f);
+ conf.setInt(YarnConfiguration.NM_VCORES, 8);
+ containersMonitor.init(conf);
+ when(mockContainerScheduler.getContainersMonitor())
+ .thenReturn(containersMonitor);
+ }
+
+ /**
+ * Node has capacity for 1024 MB and 8 cores. Saturate the node. When full the
+ * hasResourceAvailable should return false.
+ */
+ @Test
+ public void testHasResourcesAvailable() {
+ AllocationBasedResourceUtilizationTracker tracker =
+ new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
+ Container testContainer = mock(Container.class);
+ when(testContainer.getResource()).thenReturn(Resource.newInstance(512, 4));
+ for (int i = 0; i < 2; i++) {
+ Assert.assertTrue(tracker.hasResourcesAvailable(testContainer));
+ tracker.addContainerResources(testContainer);
+ }
+ Assert.assertFalse(tracker.hasResourcesAvailable(testContainer));
+ }
+
+ /**
+ * Test the case where the current allocation has been truncated to 0.8888891
+ * (8/9 cores used). Request 1 additional core - hasEnoughCpu should return
+ * true.
+ */
+ @Test
+ public void testHasEnoughCpu() {
+ AllocationBasedResourceUtilizationTracker tracker =
+ new AllocationBasedResourceUtilizationTracker(mockContainerScheduler);
+ float currentAllocation = 0.8888891f;
+ long totalCores = 9;
+ int alreadyUsedCores = 8;
+ Assert.assertTrue(tracker.hasEnoughCpu(currentAllocation, totalCores,
+ (int) totalCores - alreadyUsedCores));
+ Assert.assertFalse(tracker.hasEnoughCpu(currentAllocation, totalCores,
+ (int) totalCores - alreadyUsedCores + 1));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/24] hadoop git commit: YARN-8809. Refactor AbstractYarnScheduler
and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo
Chen via asuresh)
Posted by ha...@apache.org.
YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d217ee3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d217ee3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d217ee3
Branch: refs/heads/YARN-1011
Commit: 8d217ee3c38c18bc1929cfd981a7296898d4a202
Parents: 0a54014
Author: Arun Suresh <as...@apache.org>
Authored: Fri Sep 21 12:02:00 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:03 2018 -0700
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 32 ++++++++------------
.../scheduler/capacity/CapacityScheduler.java | 25 ++++++++++++++-
.../scheduler/fair/FairScheduler.java | 11 ++++---
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 25 +++++++++++++++
5 files changed, 70 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a8f5093..63f534f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -677,25 +677,12 @@ public abstract class AbstractYarnScheduler
}
if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
- completedContainerInternal(rmContainer, containerStatus, event);
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
completeOustandingUpdatesWhichAreReserved(
rmContainer, containerStatus, event);
} else {
- ContainerId containerId = rmContainer.getContainerId();
- // Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(containerId, containerStatus, event));
- SchedulerApplicationAttempt schedulerAttempt =
- getCurrentAttemptForContainer(containerId);
- if (schedulerAttempt != null) {
- schedulerAttempt.removeRMContainer(containerId);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Completed container: " + rmContainer.getContainerId() +
- " in state: " + rmContainer.getState() + " event:" + event);
- }
- getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
- rmContainer.getContainerId(), false);
+ completeOpportunisticContainerInternal(rmContainer, containerStatus,
+ event);
}
// If the container is getting killed in ACQUIRED state, the requester (AM
@@ -705,6 +692,12 @@ public abstract class AbstractYarnScheduler
recoverResourceRequestForContainer(rmContainer);
}
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
+ }
+
// Optimization:
// Check if there are in-flight container updates and complete the
// associated temp containers. These are removed when the app completes,
@@ -722,7 +715,7 @@ public abstract class AbstractYarnScheduler
.getReservedSchedulerKey().getContainerToUpdate();
if (containerToUpdate != null &&
containerToUpdate.equals(containerStatus.getContainerId())) {
- completedContainerInternal(resContainer,
+ completeGuaranteedContainerInternal(resContainer,
ContainerStatus.newInstance(resContainer.getContainerId(),
containerStatus.getState(), containerStatus
.getDiagnostics(),
@@ -732,8 +725,9 @@ public abstract class AbstractYarnScheduler
}
}
- // clean up a completed container
- protected abstract void completedContainerInternal(RMContainer rmContainer,
+ // clean up a completed guaranteed container
+ protected abstract void completeGuaranteedContainerInternal(
+ RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event);
protected void releaseContainers(List<ContainerId> containers,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 0b7115b..c06122f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -2066,7 +2067,29 @@ public class CapacityScheduler extends
}
@Override
- protected void completedContainerInternal(
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ ContainerId containerId = rmContainer.getContainerId();
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerFinishedEvent(containerId, containerStatus, event));
+ SchedulerApplicationAttempt schedulerAttempt =
+ getCurrentAttemptForContainer(containerId);
+ if (schedulerAttempt != null) {
+ schedulerAttempt.removeRMContainer(containerId);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Completed OPPORTUNISTIC container: " +
+ rmContainer.getContainerId() + " in state: " +
+ rmContainer.getState() + " event:" + event);
+ }
+ getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
+ rmContainer.getContainerId(), false);
+ }
+
+ @Override
+ protected void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
Container container = rmContainer.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 31f1961..744776a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -719,11 +719,14 @@ public class FairScheduler extends
}
}
- /**
- * Clean up a completed container.
- */
@Override
- protected void completedContainerInternal(
+ protected void completeGuaranteedContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeContainerInternal(rmContainer, containerStatus, event);
+ }
+
+ private void completeContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 8396db5..76648c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -836,7 +836,7 @@ public class FifoScheduler extends
@Lock(FifoScheduler.class)
@Override
- protected synchronized void completedContainerInternal(
+ protected synchronized void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 7fbf84a..1ac20f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3194,6 +3194,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("unexpected container execution type",
ExecutionType.GUARANTEED,
allocatedContainers1.get(0).getExecutionType());
+ assertEquals(1,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
// node utilization is low after the container is launched on the node
ContainerStatus containerStatus = ContainerStatus.newInstance(
@@ -3235,6 +3237,29 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// OPPORTUNISTIC container allocation.
assertTrue("No reservation should be made.",
scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512 + 1024,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
+
+ // now the OPPORTUNISTIC container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers3.get(0).getId(),
+ ContainerState.COMPLETE, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
} finally {
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/24] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index a6b30b3..ad3413f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,17 +23,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang3.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.log4j.Level;
@@ -84,13 +80,13 @@ public class TestContainerResourceUsage {
RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
- "Before app submittion, memory seconds should have been 0 but was "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() == 0);
+ "Before app submission, memory seconds should have been 0 but was "
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() == 0);
Assert.assertTrue(
"Before app submission, vcore seconds should have been 0 but was "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() == 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
@@ -105,7 +101,8 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
+ while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0
+ && cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
@@ -113,27 +110,35 @@ public class TestContainerResourceUsage {
rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
"While app is running, memory seconds should be >0 but is "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() > 0);
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() > 0);
Assert.assertTrue(
"While app is running, vcore seconds should be >0 but is "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() > 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() > 0);
MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ ru.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ ru.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ ru.getOpportunisticMemorySeconds(),
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ ru.getOpportunisticVcoreSeconds(),
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
}
- @Test (timeout = 120000)
+ @Test ()
public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
@@ -191,7 +196,7 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (app0.getRMAppMetrics().getMemorySeconds() <= 0
+ while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0
&& cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
@@ -216,19 +221,29 @@ public class TestContainerResourceUsage {
rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
// Check that the container metrics match those from the app usage report.
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
for (RMContainer c : rmContainers) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, metricsBefore.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, metricsBefore.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ metricsBefore.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ metricsBefore.getOpportunisticVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
@@ -238,10 +253,22 @@ public class TestContainerResourceUsage {
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
- Assert.assertEquals("Vcore seconds were not the same after RM Restart",
- metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
- Assert.assertEquals("Memory seconds were not the same after RM Restart",
- metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+ Assert.assertEquals(
+ "Guaranteed vcore seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedVcoreSeconds(),
+ metricsAfter.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(
+ "Guaranteed memory seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedMemorySeconds(),
+ metricsAfter.getGuaranteedMemorySeconds());
+ Assert.assertEquals(
+ "Opportunistic vcore seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticVcoreSeconds(),
+ metricsAfter.getOpportunisticVcoreSeconds());
+ Assert.assertEquals(
+ "Opportunistic memory seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticMemorySeconds(),
+ metricsAfter.getOpportunisticMemorySeconds());
rm0.stop();
rm0.close();
@@ -264,10 +291,8 @@ public class TestContainerResourceUsage {
MockRM rm = new MockRM(conf);
rm.start();
- RMApp app =
- rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
- null, "MAPREDUCE", false, keepRunningContainers);
+ RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false,
+ "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
MockNM nm =
new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
nm.registerNode();
@@ -275,18 +300,17 @@ public class TestContainerResourceUsage {
MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
int NUM_CONTAINERS = 1;
// allocate NUM_CONTAINERS containers
- am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
List<Container> containers =
- am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ am0.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
- containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers());
+ containers.addAll(am0.allocate(new ArrayList<>(),
+ new ArrayList<>()).getAllocatedContainers());
Thread.sleep(200);
}
@@ -294,26 +318,27 @@ public class TestContainerResourceUsage {
ContainerId containerId2 =
ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- containerId2.getContainerId(), ContainerState.RUNNING);
+ containerId2.getContainerId(), ContainerState.RUNNING);
rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
- Collection<RMContainer> rmContainers =
- rm.scheduler
- .getSchedulerAppInfo(am0.getApplicationAttemptId())
- .getLiveContainers();
+ Collection<RMContainer> rmContainers = rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId =
app.getCurrentAppAttempt().getMasterContainer().getId();
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- amContainerId.getContainerId(), ContainerState.COMPLETE);
+ amContainerId.getContainerId(), ContainerState.COMPLETE);
rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.drainEvents();
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers) {
@@ -321,8 +346,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
if (c.getContainerId().equals(amContainerId)) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
} else {
// The remaining container should be RUNNING.
Assert.assertTrue("After first attempt failed, remaining container "
@@ -336,8 +363,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
}
@@ -355,14 +384,12 @@ public class TestContainerResourceUsage {
am1.registerAppAttempt();
rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// allocate NUM_CONTAINERS containers
- am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
- containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ containers = am1.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
@@ -388,19 +415,26 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
- return;
}
private void waitforContainerCompletion(MockRM rm, MockNM nm,
@@ -419,16 +453,10 @@ public class TestContainerResourceUsage {
private AggregateAppResourceUsage calculateContainerResourceMetrics(
RMContainer rmContainer) {
- Resource resource = rmContainer.getContainer().getResource();
- long usedMillis =
- rmContainer.getFinishTime() - rmContainer.getCreationTime();
- long memorySeconds = resource.getMemorySize()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- long vcoreSeconds = resource.getVirtualCores()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- Map<String, Long> map = new HashMap<>();
- map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- return new AggregateAppResourceUsage(map);
+ ContainerResourceUsageReport resourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ return new AggregateAppResourceUsage(
+ resourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ resourceUsageReport.getOpportunisticResourceSecondsMap());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 342dab8..58be457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -209,7 +209,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
}
@Override
@@ -375,7 +375,7 @@ public abstract class MockAsm extends MockApps {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
- new HashMap<>());
+ new HashMap<>(), null, new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
index 5b0c34f..cdfd361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
@@ -479,7 +479,7 @@ public class TestCombinedSystemMetricsPublisher {
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(rmApp.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap,
- preemptedMap));
+ preemptedMap, new HashMap<>()));
when(rmApp.getApplicationTags()).thenReturn(
Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index f2d82e9..a8f31e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
- app.getRMAppMetrics().getMemorySeconds(),
+ app.getRMAppMetrics().getGuaranteedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getGuaranteedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
- app.getRMAppMetrics().getVcoreSeconds(),
+ app.getRMAppMetrics().getOpportunisticMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getOpportunisticVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
app.getRMAppMetrics().getPreemptedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
@@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher {
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
- .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
+ .thenReturn(new RMAppMetrics(null, 0, 0,
+ resourceMap, preemptedMap, new HashMap<>()));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 3b503e5..921d90c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0);
}
@Test(timeout = 10000)
@@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
- resourceSecondsMap
+ guaranteedResourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
- new HashMap<>()));
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 3454d72..4f84567 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -202,7 +202,8 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -300,7 +301,8 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -377,7 +379,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -402,7 +405,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 11be3b1..12b6033 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -555,7 +555,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<>(),
+ new HashMap<>(0)));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -567,7 +568,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, new HashMap<>(), new HashMap<>());
+ finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -804,20 +805,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), 0L);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- 0L);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
+ Map<String, Long> guaranteedResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> opportunisticResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> preemptedResoureSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
+ amExitStatus, 0, guaranteedResourceSecondsMap,
+ preemptedResoureSecondsMap, opportunisticResourceSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 70887e0..6ee5e40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -243,7 +242,7 @@ public class TestRMAppTransitions {
ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
doReturn(null).when(resourceScheduler)
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ .getAppActiveResourceUsageReport(Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
@@ -1249,7 +1248,8 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
- Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
+ Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT,
+ report.getApplicationResourceUsageReport());
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
Assert.assertTrue("bad proxy url for app",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..550e24d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions {
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L);
when(resourceScheduler
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any()))
.thenReturn(appResUsgRpt);
spyRMContext = spy(rmContext);
Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
@@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions {
ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
- when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
- .thenReturn(appResUsgRpt);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppActiveResourceUsageReport(
+ any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
// start and finish the attempt
Container amContainer = allocateApplicationAttempt();
@@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions {
// expect usage stats to come from the scheduler report
ApplicationResourceUsageReport report =
applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(123456L, report.getMemorySeconds());
- Assert.assertEquals(55544L, report.getVcoreSeconds());
+ Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds());
+ Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds());
// finish app attempt and remove it from scheduler
- when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions {
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
report = applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(223456, report.getMemorySeconds());
- Assert.assertEquals(75544, report.getVcoreSeconds());
+ Assert.assertEquals(223456, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index def0419..943a59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -2838,7 +2838,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Thread.sleep(1000);
AllocateResponse allocResponse = am1.schedule();
ApplicationResourceUsageReport report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
@@ -2851,7 +2851,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
allocResponse = am1.schedule();
}
report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index e239191..7a10d19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate {
RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
ApplicationResourceUsageReport appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(1024, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(1, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
// request a container.
am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(2048, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(2, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
LeafQueue queue =
(LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
.getQueue("a");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index f88ac8b..de97ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -66,7 +66,7 @@ public class TestAppPage {
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 93377be..3f5926a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Token;
@@ -259,7 +258,7 @@ public class TestRMWebApp {
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getLaunchTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
- (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
+ null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index cea0088..87846c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
- new HashMap<>(), new HashMap<>());
+ new HashMap<>(), new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 15f94e1..8eb6023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1605,7 +1605,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 45 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
index 83e0056..d034abb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
@@ -222,7 +222,7 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
}
private int getExpectedNumberOfElements(RMApp app) {
- int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
+ int expectedNumberOfElements = 45 + 2; // 2 -> resourceRequests
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
expectedNumberOfElements++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 40bdbd8..6f1ee3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -339,12 +339,22 @@ public final class RouterWebServiceUtil {
am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB());
am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores());
+ am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() +
+ uam.getAllocatedOpportunisticMB());
+ am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() +
+ uam.getAllocatedOpportunisticVCores());
am.setReservedMB(am.getReservedMB() + uam.getReservedMB());
am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB());
am.setRunningContainers(
am.getRunningContainers() + uam.getRunningContainers());
- am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds());
- am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds());
+ am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() +
+ uam.getGuaranteedMemorySeconds());
+ am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() +
+ uam.getGuaranteedVcoreSeconds());
+ am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() +
+ uam.getOpportunisticVcoreSeconds());
+ am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() +
+ uam.getOpportunisticMemorySeconds());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804..1a5c42e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil {
Assert.assertEquals(value * 3, app.getReservedMB());
Assert.assertEquals(value * 3, app.getReservedVCores());
Assert.assertEquals(value * 3, app.getRunningContainers());
- Assert.assertEquals(value * 3, app.getMemorySeconds());
- Assert.assertEquals(value * 3, app.getVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds());
Assert.assertEquals(3, app.getResourceRequests().size());
}
@@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil {
am.getResourceRequests().add(new ResourceRequestInfo());
am.setAllocatedMB(value);
am.setAllocatedVCores(value);
+ am.setAllocatedOpportunisticVCores(value);
+ am.setAllocatedOpportunisticMB(value);
am.setReservedMB(value);
am.setReservedVCores(value);
am.setRunningContainers(value);
- am.setMemorySeconds(value);
- am.setVcoreSeconds(value);
+ am.setGuaranteedMemorySeconds(value);
+ am.setGuaranteedVcoreSeconds(value);
+ am.setOpportunisticMemorySeconds(value);
+ am.setOpportunisticVcoreSeconds(value);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/24] hadoop git commit: YARN-7491. Make sure AM is not scheduled
on an opportunistic container. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f6cac1c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f6cac1c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f6cac1c
Branch: refs/heads/YARN-1011
Commit: 6f6cac1c32c472bed3fbaefbd2070fa57e2b383b
Parents: 6d028d7
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:14 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 129 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6cac1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index ff88225..a507e8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -351,15 +351,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6cac1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6cac1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d0f2ce6..6ea55ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -516,8 +516,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -548,7 +548,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6cac1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 0bd5372..8617dc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -839,15 +839,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -869,9 +899,15 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
- req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression(RMNodeLabelsManager.NO_LABEL)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(ResourceRequest.clone(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -892,9 +928,16 @@ public class TestAppManager{
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true,
- "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -936,7 +979,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/24] hadoop git commit: YARN-8427. Don't start opportunistic
containers at container scheduler/finish event with over-allocation.
Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-8427. Don't start opportunistic containers at container scheduler/finish event with over-allocation. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98218c20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98218c20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98218c20
Branch: refs/heads/YARN-1011
Commit: 98218c208bc3d5320befb96c0771b06fa2f663a4
Parents: c1dae72
Author: Miklos Szegedi <sz...@apache.org>
Authored: Fri Jun 22 10:25:31 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:01 2018 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 6 +-
.../monitor/ContainersMonitorImpl.java | 2 +-
.../AllocationBasedResourceTracker.java | 8 +-
.../scheduler/ContainerScheduler.java | 38 +++---
.../scheduler/ResourceUtilizationTracker.java | 6 +
.../UtilizationBasedResourceTracker.java | 2 +-
...estContainerSchedulerWithOverAllocation.java | 126 +++++++++++++++++--
7 files changed, 155 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a08e227..0850f74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -236,7 +236,7 @@ public class ContainerManagerImpl extends CompositeService implements
this.dirsHandler = dirsHandler;
// ContainerManager level dispatcher.
- dispatcher = new AsyncDispatcher("NM ContainerManager dispatcher");
+ dispatcher = createDispatcher();
this.deletionService = deletionContext;
this.metrics = metrics;
@@ -295,6 +295,10 @@ public class ContainerManagerImpl extends CompositeService implements
this.writeLock = lock.writeLock();
}
+ protected AsyncDispatcher createDispatcher() {
+ return new AsyncDispatcher("NM ContainerManager dispatcher");
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a045d78..8f72b56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -271,7 +271,7 @@ public class ContainersMonitorImpl extends AbstractService implements
/**
* Check all prerequisites for NM over-allocation.
*/
- private void checkOverAllocationPrerequisites() throws YarnException {
+ protected void checkOverAllocationPrerequisites() throws YarnException {
// LinuxContainerExecutor is required to enable overallocation
if (!(containerExecutor instanceof LinuxContainerExecutor)) {
throw new YarnException(LinuxContainerExecutor.class.getName() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
index 86b3698..a3e19eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceTracker.java
@@ -56,11 +56,8 @@ public class AllocationBasedResourceTracker
return this.containersAllocation;
}
- /**
- * Get the amount of resources that have not been allocated to containers.
- * @return Resource resources that have not been allocated to containers.
- */
- protected Resource getUnallocatedResources() {
+ @Override
+ public Resource getUnallocatedResources() {
// unallocated resources = node capacity - containers allocation
// = -(container allocation - node capacity)
ResourceUtilization allocationClone =
@@ -80,7 +77,6 @@ public class AllocationBasedResourceTracker
return unallocated;
}
-
@Override
public Resource getAvailableResources() {
return getUnallocatedResources();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0bebe44..0d7de67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -218,7 +218,9 @@ public class ContainerScheduler extends AbstractService implements
startPendingContainers(false);
break;
case SCHEDULE_CONTAINERS:
- startPendingContainers(true);
+ // try to launch OPPORTUNISTIC containers allowing over-allocation if the
+ // node utilization is low.
+ startOpportunisticContainers(utilizationTracker.getAvailableResources());
break;
default:
LOG.error("Unknown event arrived at ContainerScheduler: "
@@ -410,11 +412,7 @@ public class ContainerScheduler extends AbstractService implements
this.metrics.completeOpportunisticContainer(container.getResource());
}
- // In case of over-allocation being turned on, we may need to reclaim
- // more resources since the opportunistic containers that have been
- // killed or paused may have not released as much resource as we need.
- boolean reclaimOpportunisticResources = context.isOverAllocationEnabled();
- startPendingContainers(reclaimOpportunisticResources);
+ startPendingContainers(false);
}
}
@@ -434,22 +432,32 @@ public class ContainerScheduler extends AbstractService implements
return;
}
+ boolean guaranteedContainersLaunched = startGuaranteedContainers(
+ reclaimOpportunisticResources);
+
+ if (guaranteedContainersLaunched) {
+ // if all GUARANTEED containers are launched, try to launch OPPORTUNISTIC
+ // containers with unallocated resources only.
+ startOpportunisticContainers(
+ utilizationTracker.getUnallocatedResources());
+ }
+ }
+
+ private boolean startGuaranteedContainers(
+ boolean reclaimOpportunisticResources) {
Resource available = utilizationTracker.getAvailableResources();
// Start guaranteed containers that are queued, if resources available.
boolean allGuaranteedContainersLaunched =
startGuaranteedContainers(available);
- // Start opportunistic containers, if resources available, which is true
- // if all guaranteed containers in queue have been launched.
- if (allGuaranteedContainersLaunched) {
- startOpportunisticContainers(available);
- } else {
+
+ if (!allGuaranteedContainersLaunched && reclaimOpportunisticResources) {
// If not all guaranteed containers in queue are launched, we may need
// to reclaim resources from opportunistic containers that are running.
- if (reclaimOpportunisticResources) {
- reclaimOpportunisticContainerResources();
- }
+ reclaimOpportunisticContainerResources();
}
+
+ return allGuaranteedContainersLaunched;
}
/**
@@ -693,8 +701,6 @@ public class ContainerScheduler extends AbstractService implements
*
* If the node is over-allocating itself, this may cause not enough
* OPPORTUNISTIC containers being killed/paused in cases where the running
- * OPPORTUNISTIC containers are not consuming fully their resource requests.
- * We'd check again upon container completion events to see if more running
* OPPORTUNISTIC containers need to be killed/paused.
*
* @return the amount of resource needed to be reclaimed for this container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 98d99c6..7a7c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -45,6 +45,12 @@ public interface ResourceUtilizationTracker {
Resource getAvailableResources();
/**
+ * Get the amount of resources left un-allocated.
+ * @return Resource unallocated resources
+ */
+ Resource getUnallocatedResources();
+
+ /**
* Add Container's resources to Node Utilization upon container launch.
* @param container Container.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
index 6f9bc82..f976700 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/UtilizationBasedResourceTracker.java
@@ -61,7 +61,7 @@ public class UtilizationBasedResourceTracker
@Override
public Resource getAvailableResources() {
- Resource resourceBasedOnAllocation = getUnallocatedResources();
+ Resource resourceBasedOnAllocation = super.getUnallocatedResources();
Resource resourceBasedOnUtilization =
getResourcesAvailableBasedOnUtilization();
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98218c20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
index 384b116..183d868 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
@@ -114,6 +115,9 @@ public class TestContainerSchedulerWithOverAllocation
conf.setFloat(
YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
0.75f);
+ // disable the monitor thread in ContainersMonitor to allow control over
+ // when opportunistic containers are launched with over-allocation
+ conf.setBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED, false);
super.setup();
}
@@ -190,6 +194,27 @@ public class TestContainerSchedulerWithOverAllocation
BuilderUtils.newResource(512, 1), false))
));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // this container is not expected to be started immediately because
+ // opportunistic containers cannot be started if the node would be
+ // over-allocated
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // try to start opportunistic containers out of band.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
// this container is expected to be started immediately because there
// are (memory: 1024, vcore: 0.625) available based on over-allocation
BaseContainerManagerTest.waitForContainerSubState(
@@ -244,6 +269,16 @@ public class TestContainerSchedulerWithOverAllocation
createStartContainerRequest(2,
BuilderUtils.newResource(512, 1), false))
));
+
+ // try to start opportunistic containers out of band because they can
+ // not be launched at container scheduler event if the node would be
+ // over-allocated.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
// this container will not start immediately because there is not
// enough resource available at the moment either in terms of
// resources unallocated or in terms of the actual utilization
@@ -299,6 +334,27 @@ public class TestContainerSchedulerWithOverAllocation
BuilderUtils.newResource(512, 1), false))
));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // this container is not expected to be started immediately because
+ // opportunistic containers cannot be started if the node would be
+ // over-allocated
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // try to start opportunistic containers out of band.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
// this container is expected to be started because there is resources
// available because the actual utilization is very low
BaseContainerManagerTest.waitForContainerSubState(containerManager,
@@ -355,6 +411,9 @@ public class TestContainerSchedulerWithOverAllocation
containerManager.startContainers(
StartContainersRequest.newInstance(moreContainerRequests));
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
// All OPPORTUNISTIC containers but the last one should be queued.
// The last OPPORTUNISTIC container to launch should be killed.
BaseContainerManagerTest.waitForContainerState(
@@ -580,13 +639,27 @@ public class TestContainerSchedulerWithOverAllocation
}
}
));
- // All three GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // Two OPPORTUNISTIC containers are expected to start with the
+ // unallocated resources, but one will be queued because no
+ // over-allocation is allowed at container scheduler events.
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(0), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(1), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // try to start the opportunistic container out of band because it can
+ // not be launched at container scheduler event if the node would be
+ // over-allocated.
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // now the queued opportunistic container should also start
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.RUNNING);
// the containers utilization is low
@@ -625,7 +698,6 @@ public class TestContainerSchedulerWithOverAllocation
* and fourth OPPORTUNISTIC container (which releases no resources) and
* then the second OPPORTUNISTIC container.
*/
- @Test
public void
testKillOppContainersConservativelyWithOverallocationHighUtilization()
throws Exception {
@@ -730,7 +802,7 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.RUNNING);
- // the contianers utilization is at the overallocation threshold
+ // the container utilization is at the overallocation threshold
setContainerResourceUtilization(
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
@@ -740,7 +812,7 @@ public class TestContainerSchedulerWithOverAllocation
add(createStartContainerRequest(3,
BuilderUtils.newResource(512, 1), false));
add(createStartContainerRequest(4,
- BuilderUtils.newResource(512, 1), false));
+ BuilderUtils.newResource(800, 1), false));
}
}
));
@@ -759,12 +831,35 @@ public class TestContainerSchedulerWithOverAllocation
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(2), ContainerSubState.DONE);
- // the two OPPORTUNISTIC containers are expected to start together
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+
+ // only one OPPORTUNISTIC container is start because no over-allocation
+ // is allowed to start OPPORTUNISTIC containers at container finish event.
BaseContainerManagerTest.waitForContainerSubState(containerManager,
createContainerId(3), ContainerSubState.RUNNING);
BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
+ createContainerId(4), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+ // now try to start the OPPORTUNISTIC container that was queued because
+ // we don't start OPPORTUNISTIC containers at container finish event if
+ // the node would be over-allocated
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+ ((LongRunningContainerSimulatingContainersManager) containerManager)
+ .drainAsyncEvents();
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
{
put(createContainerId(0), ContainerSubState.RUNNING);
@@ -817,7 +912,7 @@ public class TestContainerSchedulerWithOverAllocation
setContainerResourceUtilization(
ResourceUtilization.newInstance(1536, 0, 1.0f/2));
- // try to start containers out of band.
+ // try to start opportunistic containers out of band.
((LongRunningContainerSimulatingContainersManager)containerManager)
.startContainersOutOfBandUponLowUtilization();
@@ -937,6 +1032,11 @@ public class TestContainerSchedulerWithOverAllocation
return ugi;
}
+ @Override
+ protected AsyncDispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+
/**
* Create a container launcher that signals container processes
* with a dummy pid. The container processes are simulated in
@@ -977,6 +1077,10 @@ public class TestContainerSchedulerWithOverAllocation
((ContainerMonitorForOverallocationTest) getContainersMonitor())
.attemptToStartContainersUponLowUtilization();
}
+
+ public void drainAsyncEvents() {
+ ((DrainDispatcher) dispatcher).await();
+ }
}
/**
@@ -1113,6 +1217,12 @@ public class TestContainerSchedulerWithOverAllocation
containerResourceUsage, System.currentTimeMillis());
}
+ @Override
+ protected void checkOverAllocationPrerequisites() {
+ // do not check
+ }
+
+
public void setContainerResourceUsage(
ResourceUtilization containerResourceUsage) {
this.containerResourceUsage = containerResourceUsage;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/24] hadoop git commit: Revert "YARN-6675. Add NM support to
launch opportunistic containers based on overallocation. Contributed by Haibo
Chen."
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index f24df38..6b3ac67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -67,8 +66,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceTracker
- allocationBasedResourceTracker;
+ @Mock private AllocationBasedResourceUtilizationTracker
+ allocationBasedResourceUtilizationTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -86,13 +85,12 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
- when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceTracker)
- .containerLaunched(container);
+ doNothing().when(allocationBasedResourceUtilizationTracker)
+ .addContainerResources(container);
}
@After public void tearDown() {
@@ -114,8 +112,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -134,8 +132,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -154,8 +152,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -174,8 +172,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -194,8 +192,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -214,8 +212,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -234,8 +232,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -254,8 +252,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -274,8 +272,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -294,8 +292,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -313,8 +311,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -332,7 +330,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
- .containerLaunched(container);
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8851333/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
deleted file mode 100644
index 384b116..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
+++ /dev/null
@@ -1,1121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ContainerSubState;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.exceptions.ConfigurationException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-/**
- * Test ContainerScheduler behaviors when NM overallocation is turned on.
- */
-public class TestContainerSchedulerWithOverAllocation
- extends BaseContainerManagerTest {
- private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
- private static final int NM_CONTAINERS_VCORES = 4;
- private static final int NM_CONTAINERS_MEMORY_MB = 2048;
-
- static {
- LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
- }
-
- public TestContainerSchedulerWithOverAllocation()
- throws UnsupportedFileSystemException {
- }
-
- @Override
- protected ContainerExecutor createContainerExecutor() {
- DefaultContainerExecutor exec =
- new LongRunningContainerSimulatingContainerExecutor();
- exec.setConf(conf);
- return exec;
- }
-
- @Override
- protected ContainerManagerImpl createContainerManager(
- DeletionService delSrvc) {
- return new LongRunningContainerSimulatingContainersManager(
- context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
- }
-
- @Override
- public void setup() throws IOException {
- conf.setInt(
- YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
- NM_OPPORTUNISTIC_QUEUE_LIMIT);
- conf.setFloat(
- YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
- 0.75f);
- conf.setFloat(
- YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
- 0.75f);
- super.setup();
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
- * not exceed the capacity of the node. Both containers are expected to start
- * running immediately.
- */
- @Test
- public void testStartMultipleContainersWithoutOverallocation()
- throws Exception {
- containerManager.start();
-
- StartContainersRequest allRequests = StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() { {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- } }
- );
- containerManager.startContainers(allRequests);
-
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
- * is very low relative to their resource request, resulting in a low node
- * utilization. Then start another OPPORTUNISTIC containers which requests
- * more than what's left unallocated on the node. Due to overallocation
- * being turned on and node utilization being low, the second OPPORTUNISTIC
- * container is also expected to be launched immediately.
- */
- @Test
- public void testStartOppContainersWithPartialOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- // the current containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- // start a container that requests more than what's left unallocated
- // 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
-
- // this container is expected to be started immediately because there
- // are (memory: 1024, vcore: 0.625) available based on over-allocation
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
- * of the resources they requested, resulting in a high node utilization.
- * Then start another OPPORTUNISTIC containers which requests more than what's
- * left unallocated on the node. Because of the high resource utilization on
- * the node, the projected utilization, if we were to start the second
- * OPPORTUNISTIC container immediately, will go over the NM overallocation
- * threshold, so the second OPPORTUNISTIC container is expected to be queued.
- */
- @Test
- public void testQueueOppContainerWithPartialOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(
- containerManager, createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers utilization is high
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1500, 0, 1.0f/8));
-
- // start a container that requests more than what's left unallocated
- // 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
- // this container will not start immediately because there is not
- // enough resource available at the moment either in terms of
- // resources unallocated or in terms of the actual utilization
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.SCHEDULED);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.SCHEDULED);
- }
- });
- }
-
- /**
- * Start two GUARANTEED containers which in aggregate takes up the whole node
- * capacity, yet whose utilization is low relative to their resource request,
- * resulting in a low node resource utilization. Then try to start another
- * OPPORTUNISTIC containers. Because the resource utilization across the node
- * is low and overallocation being turned on, the OPPORTUNISTIC container is
- * expected to be launched immediately even though there is no resources left
- * unallocated.
- */
- @Test
- public void testStartOppContainersWithOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the current containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(800, 0, 1.0f/8));
-
- // start a container when there is no resources left unallocated.
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false))
- ));
-
- // this container is expected to be started because there is resources
- // available because the actual utilization is very low
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
-
- /**
- * Start two GUARANTEED containers which in aggregate take up the whole node
- * capacity and fully utilize the resources they requested. Then try to start
- * four OPPORTUNISTIC containers of which three will be queued and one will be
- * killed because of the max queue length is 3.
- */
- @Test
- public void testQueueOppContainersWithFullUtilization() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers are fully utilizing their resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(2048, 0, 1.0f/8));
-
- // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
- // queue can hold when there is no unallocated resource left.
- List<StartContainerRequest> moreContainerRequests =
- new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
- for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
- moreContainerRequests.add(
- createStartContainerRequest(2 + a,
- BuilderUtils.newResource(512, 1), false));
- }
- containerManager.startContainers(
- StartContainersRequest.newInstance(moreContainerRequests));
-
- // All OPPORTUNISTIC containers but the last one should be queued.
- // The last OPPORTUNISTIC container to launch should be killed.
- BaseContainerManagerTest.waitForContainerState(
- containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
- ContainerState.COMPLETE);
-
- HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
- new HashMap<>();
- expectedContainerStatus.put(
- createContainerId(0), ContainerSubState.RUNNING);
- expectedContainerStatus.put(
- createContainerId(1), ContainerSubState.RUNNING);
- expectedContainerStatus.put(
- createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
- ContainerSubState.DONE);
- for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
- expectedContainerStatus.put(
- createContainerId(i + 2), ContainerSubState.SCHEDULED);
- }
- verifyContainerStatuses(expectedContainerStatus);
- }
-
- /**
- * Start two GUARANTEED containers that together does not take up the
- * whole node. Then try to start one OPPORTUNISTIC container that will
- * fit into the remaining unallocated space on the node.
- * The OPPORTUNISTIC container is expected to start even though the
- * current node utilization is above the NM overallocation threshold,
- * because it's always safe to launch containers as long as the node
- * has not been fully allocated.
- */
- @Test
- public void testStartOppContainerWithHighUtilizationNoOverallocation()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1200, 1), true));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(400, 1), true));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // containers utilization is above the over-allocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1600, 0, 1.0f/2));
-
- // start a container that can just fit in the remaining unallocated space
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(400, 1), false))
- ));
-
- // the OPPORTUNISTIC container can be safely launched even though
- // the container utilization is above the NM overallocation threshold
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start two OPPORTUNISTIC containers first whose utilization is low relative
- * to the resources they requested, resulting in a low node utilization. Then
- * try to start a GUARANTEED container which requests more than what's left
- * unallocated on the node. Because the node utilization is low and NM
- * overallocation is turned on, the GUARANTEED container is expected to be
- * started immediately without killing any running OPPORTUNISTIC containers.
- */
- @Test
- public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node: (512 + 1024 + 824) > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- // the GUARANTEED container is expected be launched immediately without
- // killing any OPPORTUNISTIC containers.
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start two OPPORTUNISTIC containers whose utilization will be high relative
- * to the resources they requested, resulting in a high node utilization.
- * Then try to start a GUARANTEED container which requests more than what's
- * left unallocated on the node. Because the node is under high utilization,
- * the second OPPORTUNISTIC container is expected to be killed in order to
- * make room for the GUARANTEED container.
- */
- @Test
- public void testKillOppContainersWithPartialOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(824, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
-
- // the containers utilization is very high
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1800, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node 512 + 1024 + 824 > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- // the last launched OPPORTUNISTIC container is expected to be killed
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.DONE);
-
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(1))) {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- } else {
- Assert.assertEquals(status.getContainerId() + " is not RUNNING",
- ContainerSubState.RUNNING, status.getContainerSubState());
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
-
- /**
- * Start three OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node, yet whose utilization is low relative
- * to the resources they requested, resulting in a low node utilization.
- * Then try to start a GUARANTEED container. Even though the node has
- * nothing left unallocated, it is expected to start immediately
- * without killing any running OPPORTUNISTIC containers because the node
- * utilization is very low and overallocation is turned on.
- */
- @Test
- public void testKillNoOppContainersWithOverallocationLowUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All three GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- // the containers utilization is low
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1024, 0, 1.0f/8));
-
- // start a GUARANTEED container that requests more than what's left
- // unallocated on the node: (512 + 1024 + 824) > 2048
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), true))
- ));
-
- // the GUARANTEED container is expected be launched immediately without
- // killing any OPPORTUNISTIC containers
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- put(createContainerId(3), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start four OPPORTUNISTIC containers which in aggregates exceeds the
- * capacity of the node. The real resource utilization of the first two
- * OPPORTUNISTIC containers are high whereas that of the latter two are
- * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
- * container will eventually start running after preempting the third
- * and fourth OPPORTUNISTIC container (which releases no resources) and
- * then the second OPPORTUNISTIC container.
- */
- @Test
- public void
- testKillOppContainersConservativelyWithOverallocationHighUtilization()
- throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(1024, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(1024, 1), false));
- }
- }
- ));
- // All four GUARANTEED containers are all expected to start
- // because the containers utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
-
- // the containers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start a GUARANTEED container when there's nothing left unallocated
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(4,
- BuilderUtils.newResource(1024, 1), true))
- ));
-
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
- GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
- newInstance(new ArrayList<ContainerId>() {
- {
- add(createContainerId(0));
- add(createContainerId(1));
- add(createContainerId(2));
- add(createContainerId(3));
- add(createContainerId(4));
- }
- });
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
- for (ContainerStatus status : containerStatuses) {
- if (status.getContainerId().equals(createContainerId(0)) ||
- status.getContainerId().equals(createContainerId(4))) {
- Assert.assertEquals(
- ContainerSubState.RUNNING, status.getContainerSubState());
- } else {
- Assert.assertTrue(status.getDiagnostics().contains(
- "Container Killed to make room for Guaranteed Container"));
- }
- System.out.println("\nStatus : [" + status + "]\n");
- }
- }
-
- /**
- * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
- * which in aggregate exceeds the capacity of the node. The first two
- * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
- * one utilizes nearly all of its resource requested. Then try to start two
- * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
- * expected to be queued immediately. Upon the completion of the
- * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
- * are expected to start.
- */
- @Test
- public void testStartOppContainersUponContainerCompletion() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(0,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(1024, 1), true));
- }
- }
- ));
-
- // All three containers are all expected to start immediately
- // because the node utilization is low (0 at the point)
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- // the contianers utilization is at the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(3,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(4,
- BuilderUtils.newResource(512, 1), false));
- }
- }
- ));
- // the two new OPPORTUNISTIC containers are expected to be queued
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(3), ContainerSubState.SCHEDULED);
- put(createContainerId(4), ContainerSubState.SCHEDULED);
- }
- });
-
- // the GUARANTEED container is completed releasing resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(100, 0, 1.0f/5));
- allowContainerToSucceed(2);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.DONE);
-
- // the two OPPORTUNISTIC containers are expected to start together
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(3), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(4), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.DONE);
- put(createContainerId(3), ContainerSubState.RUNNING);
- put(createContainerId(4), ContainerSubState.RUNNING);
- }
- });
- }
-
- /**
- * Start one GUARANTEED container that consumes all the resources on the
- * node and keeps running, followed by two OPPORTUNISTIC containers that
- * will be queued forever because there is no containers starting or
- * finishing. Then try to start OPPORTUNISTIC containers out of band.
- */
- @Test
- public void testStartOpportunisticContainersOutOfBand() throws Exception {
- containerManager.start();
-
- containerManager.startContainers(StartContainersRequest.newInstance(
- Collections.singletonList(
- createStartContainerRequest(0,
- BuilderUtils.newResource(2048, 4), true))));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(0), ContainerSubState.RUNNING);
-
- // the container is fully utilizing its resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(2048, 0, 1.0f));
-
- // send two OPPORTUNISTIC container requests that are expected to be queued
- containerManager.startContainers(StartContainersRequest.newInstance(
- new ArrayList<StartContainerRequest>() {
- {
- add(createStartContainerRequest(1,
- BuilderUtils.newResource(512, 1), false));
- add(createStartContainerRequest(2,
- BuilderUtils.newResource(512, 1), false));
- }
- }
- ));
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.SCHEDULED);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.SCHEDULED);
-
- // the containers utilization dropped to the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(1536, 0, 1.0f/2));
-
- // try to start containers out of band.
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- // no containers in queue are expected to be launched because the
- // containers utilization is not below the over-allocation threshold
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.SCHEDULED);
- put(createContainerId(2), ContainerSubState.SCHEDULED);
- }
- });
-
- // the GUARANTEED container is completed releasing resources
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(100, 0, 1.0f/5));
-
- // the containers utilization dropped way below the overallocation threshold
- setContainerResourceUtilization(
- ResourceUtilization.newInstance(512, 0, 1.0f/8));
-
- ((LongRunningContainerSimulatingContainersManager)containerManager)
- .startContainersOutOfBandUponLowUtilization();
-
- // the two OPPORTUNISTIC containers are expected to be launched
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(1), ContainerSubState.RUNNING);
- BaseContainerManagerTest.waitForContainerSubState(containerManager,
- createContainerId(2), ContainerSubState.RUNNING);
-
- verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
- {
- put(createContainerId(0), ContainerSubState.RUNNING);
- put(createContainerId(1), ContainerSubState.RUNNING);
- put(createContainerId(2), ContainerSubState.RUNNING);
- }
- });
- }
-
-
- private void setContainerResourceUtilization(ResourceUtilization usage) {
- ((ContainerMonitorForOverallocationTest)
- containerManager.getContainersMonitor())
- .setContainerResourceUsage(usage);
- }
-
- private void allowContainerToSucceed(int containerId) {
- ((LongRunningContainerSimulatingContainerExecutor) this.exec)
- .containerSucceeded(createContainerId(containerId));
- }
-
-
- protected StartContainerRequest createStartContainerRequest(int containerId,
- Resource resource, boolean isGuaranteed) throws IOException {
- ContainerLaunchContext containerLaunchContext =
- recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
- ExecutionType.OPPORTUNISTIC;
- Token containerToken = createContainerToken(
- createContainerId(containerId),
- DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
- context.getContainerTokenSecretManager(),
- null, executionType);
-
- return StartContainerRequest.newInstance(
- containerLaunchContext, containerToken);
- }
-
- protected void verifyContainerStatuses(
- Map<ContainerId, ContainerSubState> expected)
- throws IOException, YarnException {
- List<ContainerId> statList = new ArrayList<>(expected.keySet());
- GetContainerStatusesRequest statRequest =
- GetContainerStatusesRequest.newInstance(statList);
- List<ContainerStatus> containerStatuses = containerManager
- .getContainerStatuses(statRequest).getContainerStatuses();
-
- for (ContainerStatus status : containerStatuses) {
- ContainerId containerId = status.getContainerId();
- Assert.assertEquals(containerId + " is in unexpected state",
- expected.get(containerId), status.getContainerSubState());
- }
- }
-
- /**
- * A container manager that sends a dummy container pid while it's cleaning
- * up running containers. Used along with
- * LongRunningContainerSimulatingContainerExecutor to simulate long running
- * container processes for testing purposes.
- */
- private static class LongRunningContainerSimulatingContainersManager
- extends ContainerManagerImpl {
-
- private final String user;
-
- LongRunningContainerSimulatingContainersManager(
- Context context, ContainerExecutor exec,
- DeletionService deletionContext,
- NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics,
- LocalDirsHandlerService dirsHandler, String user) {
- super(context, exec, deletionContext,
- nodeStatusUpdater, metrics, dirsHandler);
- this.user = user;
- }
-
- @Override
- protected UserGroupInformation getRemoteUgi() throws YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
- .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
- .getKeyId()));
- return ugi;
- }
-
- /**
- * Create a container launcher that signals container processes
- * with a dummy pid. The container processes are simulated in
- * LongRunningContainerSimulatingContainerExecutor which does
- * not write a pid file on behalf of containers to launch, so
- * the pid does not matter.
- */
- @Override
- protected ContainersLauncher createContainersLauncher(
- Context context, ContainerExecutor exec) {
- ContainerManagerImpl containerManager = this;
- return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
- this) {
- @Override
- protected ContainerLaunch createContainerLaunch(
- Application app, Container container) {
- return new ContainerLaunch(context, getConfig(), dispatcher,
- exec, app, container, dirsHandler, containerManager) {
- @Override
- protected String getContainerPid(Path pidFilePath)
- throws Exception {
- return "123";
- }
-
- };
- }
- };
- }
-
- @Override
- protected ContainersMonitor createContainersMonitor(
- ContainerExecutor exec) {
- return new ContainerMonitorForOverallocationTest(exec,
- dispatcher, context);
- }
-
- public void startContainersOutOfBandUponLowUtilization() {
- ((ContainerMonitorForOverallocationTest) getContainersMonitor())
- .attemptToStartContainersUponLowUtilization();
- }
- }
-
- /**
- * A container executor that simulates long running container processes
- * by having container launch threads sleep infinitely until it's given
- * a signal to finish with either a success or failure exit code.
- */
- private static class LongRunningContainerSimulatingContainerExecutor
- extends DefaultContainerExecutor {
- private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
- new ConcurrentHashMap<>();
-
- public void containerSucceeded(ContainerId containerId) {
- ContainerFinishLatch containerFinishLatch = containers.get(containerId);
- if (containerFinishLatch != null) {
- containerFinishLatch.toSucceed();
- }
- }
-
- public void containerFailed(ContainerId containerId) {
- ContainerFinishLatch containerFinishLatch = containers.get(containerId);
- if (containerFinishLatch != null) {
- containerFinishLatch.toFail();
- }
- }
-
- /**
- * Simulate long running container processes by having container launcher
- * threads wait infinitely for a signal to finish.
- */
- @Override
- public int launchContainer(ContainerStartContext ctx)
- throws IOException, ConfigurationException {
- ContainerId container = ctx.getContainer().getContainerId();
- containers.putIfAbsent(container, new ContainerFinishLatch(container));
-
- // simulate a long running container process by having the
- // container launch thread sleep forever until it's given a
- // signal to finish with a exit code.
- while (!containers.get(container).toProceed) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- return -1;
- }
- }
-
- return containers.get(container).getContainerExitCode();
- }
-
- /**
- * Override signalContainer() so that simulated container processes
- * are properly cleaned up.
- */
- @Override
- public boolean signalContainer(ContainerSignalContext ctx)
- throws IOException {
- containerSucceeded(ctx.getContainer().getContainerId());
- return true;
- }
-
- /**
- * A signal that container launch threads wait for before exiting
- * in order to simulate long running container processes.
- */
- private static final class ContainerFinishLatch {
- volatile boolean toProceed;
- int exitCode;
- ContainerId container;
-
- ContainerFinishLatch(ContainerId containerId) {
- exitCode = 0;
- toProceed = false;
- container = containerId;
- }
-
- void toSucceed() {
- exitCode = 0;
- toProceed = true;
- }
-
- void toFail() {
- exitCode = -101;
- toProceed = true;
- }
-
- int getContainerExitCode() {
- // read barrier of toProceed to make sure the exit code is not stale
- if (toProceed) {
- LOG.debug(container + " finished with exit code: " + exitCode);
- }
- return exitCode;
- }
- }
- }
-
- /**
- * A test implementation of container monitor that allows control of
- * current resource utilization.
- */
- private static class ContainerMonitorForOverallocationTest
- extends ContainersMonitorImpl {
-
- private ResourceUtilization containerResourceUsage =
- ResourceUtilization.newInstance(0, 0, 0.0f);
-
- ContainerMonitorForOverallocationTest(ContainerExecutor exec,
- AsyncDispatcher dispatcher, Context context) {
- super(exec, dispatcher, context);
- }
-
- @Override
- public long getPmemAllocatedForContainers() {
- return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
- }
-
- @Override
- public long getVmemAllocatedForContainers() {
- float pmemRatio = getConfig().getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- return (long) (pmemRatio * getPmemAllocatedForContainers());
- }
-
- @Override
- public long getVCoresAllocatedForContainers() {
- return NM_CONTAINERS_VCORES;
- }
-
- @Override
- public ContainersResourceUtilization getContainersUtilization(
- boolean latest) {
- return new ContainersMonitor.ContainersResourceUtilization(
- containerResourceUsage, System.currentTimeMillis());
- }
-
- public void setContainerResourceUsage(
- ResourceUtilization containerResourceUsage) {
- this.containerResourceUsage = containerResourceUsage;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/24] hadoop git commit: MAPREDUCE-6926. Allow MR jobs to opt out
of oversubscription. Contributed by Haibo Chen.
Posted by ha...@apache.org.
MAPREDUCE-6926. Allow MR jobs to opt out of oversubscription. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8757aa90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8757aa90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8757aa90
Branch: refs/heads/YARN-1011
Commit: 8757aa90a7c43ff523adb847e240efc2f10ca920
Parents: 6f6cac1
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jan 10 13:21:11 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:15 2018 -0700
----------------------------------------------------------------------
.../v2/app/rm/RMContainerRequestor.java | 48 ++---
.../v2/app/rm/TestRMContainerAllocator.java | 192 +++++++++++++++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../src/main/resources/mapred-default.xml | 8 +
4 files changed, 231 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8757aa90/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index bb3e1fa..d996690 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -111,6 +111,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
private final Set<String> blacklistRemovals = Collections
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private boolean optOutOfOversubscription;
public RMContainerRequestor(ClientService clientService, AppContext context) {
super(clientService, context);
@@ -136,9 +137,11 @@ public abstract class RMContainerRequestor extends RMCommunicator {
public ContainerRequest(ContainerRequestEvent event, Priority priority,
String nodeLabelExpression) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
- event.getRacks(), priority, nodeLabelExpression);
+ event.getRacks(), priority, System.currentTimeMillis(),
+ nodeLabelExpression);
}
+ @VisibleForTesting
public ContainerRequest(ContainerRequestEvent event, Priority priority,
long requestTimeMs) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
@@ -146,13 +149,6 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
public ContainerRequest(TaskAttemptId attemptID,
- Resource capability, String[] hosts, String[] racks,
- Priority priority, String nodeLabelExpression) {
- this(attemptID, capability, hosts, racks, priority,
- System.currentTimeMillis(), nodeLabelExpression);
- }
-
- public ContainerRequest(TaskAttemptId attemptID,
Resource capability, String[] hosts, String[] racks,
Priority priority, long requestTimeMs,String nodeLabelExpression) {
this.attemptID = attemptID;
@@ -186,6 +182,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
MRJobConfig.DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT);
LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
+ optOutOfOversubscription = conf.getBoolean(
+ MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ MRJobConfig.DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT);
+ LOG.info("optOutOfOversubscription is " + optOutOfOversubscription);
if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
+ blacklistDisablePercent
@@ -398,20 +398,20 @@ public abstract class RMContainerRequestor extends RMCommunicator {
for (String host : req.hosts) {
// Data-local
if (!isNodeBlacklisted(host)) {
- addResourceRequest(req.priority, host, req.capability,
+ addGuaranteedResourceRequest(req.priority, host, req.capability,
null);
}
}
// Nothing Rack-local for now
for (String rack : req.racks) {
- addResourceRequest(req.priority, rack, req.capability,
+ addGuaranteedResourceRequest(req.priority, rack, req.capability,
null);
}
// Off-switch
- addResourceRequest(req.priority, ResourceRequest.ANY, req.capability,
- req.nodeLabelExpression);
+ addGuaranteedResourceRequest(req.priority, ResourceRequest.ANY,
+ req.capability, req.nodeLabelExpression);
}
protected void decContainerReq(ContainerRequest req) {
@@ -430,18 +430,18 @@ public abstract class RMContainerRequestor extends RMCommunicator {
protected void addOpportunisticResourceRequest(Priority priority,
Resource capability) {
addResourceRequest(priority, ResourceRequest.ANY, capability, null,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, true);
}
- private void addResourceRequest(Priority priority, String resourceName,
- Resource capability, String nodeLabelExpression) {
+ private void addGuaranteedResourceRequest(Priority priority,
+ String resourceName, Resource capability, String nodeLabelExpression) {
addResourceRequest(priority, resourceName, capability, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, optOutOfOversubscription);
}
private void addResourceRequest(Priority priority, String resourceName,
Resource capability, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, boolean enforceExecutionType) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
if (remoteRequests == null) {
@@ -464,8 +464,8 @@ public abstract class RMContainerRequestor extends RMCommunicator {
remoteRequest.setCapability(capability);
remoteRequest.setNumContainers(0);
remoteRequest.setNodeLabelExpression(nodeLabelExpression);
- remoteRequest.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(executionType, true));
+ remoteRequest.setExecutionTypeRequest(ExecutionTypeRequest.
+ newInstance(executionType, enforceExecutionType));
reqMap.put(capability, remoteRequest);
}
remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
@@ -473,9 +473,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
- LOG.debug("addResourceRequest:" + " applicationId="
+ LOG.debug("addGuaranteedResourceRequest:" + " applicationId="
+ applicationId.getId() + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
+ + " resourceName=" + resourceName + " ExecutionType=" + executionType
+ + " enforceExecutionType=" + enforceExecutionType + " numContainers="
+ remoteRequest.getNumContainers() + " #asks=" + ask.size());
}
}
@@ -559,8 +560,9 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
}
String[] hosts = newHosts.toArray(new String[newHosts.size()]);
- ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability,
- hosts, orig.racks, orig.priority, orig.nodeLabelExpression);
+ ContainerRequest newReq = new ContainerRequest(orig.attemptID,
+ orig.capability, hosts, orig.racks, orig.priority,
+ System.currentTimeMillis(), orig.nodeLabelExpression);
return newReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8757aa90/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 427e6ea..5d4dcf0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -105,6 +105,8 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -868,6 +870,196 @@ public class TestRMContainerAllocator {
allocator.close();
}
+ /**
+ * Test A MapReduce job can be configured to opt out of oversubscription,
+ * that is, it always wait for guaranteed resources to execute its tasks.
+ * This is done by setting up a MapReduce job with 2 mappers and 1 reducers
+ * and capturing all ResourceRequests sent from the AM to RM, then checking
+ * if all ResourceRequests are guaranteed and their enforceExecutionType is
+ * true.
+ */
+ @Test
+ public void testMapReduceOptingOutOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(true);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or not enforced.");
+ }
+ }
+ }
+
+ /**
+ * Test a MapReduce job can be configured to opt in oversubscription (
+ * true by default). This is done by setting up a MapReduce job with 2
+ * mappers and 1 reducers and capturing all ResourceRequests sent from
+ * the AM to RM, then checking if all ResourceRequests are guaranteed
+ * but their enforceExecutionType is always set to false.
+ */
+ @Test
+ public void testMapReduceOptingInOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(false);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, false))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or it is enforced.");
+ }
+ }
+ }
+
+ /**
+ * Set up a mapreduce job with 2 mappers and 1 reducer and return
+ * all ResourceRequests sent from the AM to RM.
+ */
+ private List<ResourceRequest> captureResourceRequests(
+ boolean optOutOfOversubscription) throws Exception {
+ List<ResourceRequest> resourceRequests = new ArrayList<>();
+
+ final Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ optOutOfOversubscription);
+
+ // start the resource manager
+ final MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // submit an application
+ RMApp rmApp = rm.submitApp(1024);
+ rm.drainEvents();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ rm.drainEvents();
+
+ // start the MR AM and wait until it is in running state
+ MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
+ appAttemptId, 0), 2, 1, false,
+ this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+ mrApp.submit(conf);
+
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+ mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING);
+ amDispatcher.await();
+
+ // wait until all attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
+ .iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
+ }
+ amDispatcher.await();
+
+ // send map resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for both map tasks to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish both map tasks so that the reduce task can be scheduled
+ Iterator<Task> it = job.getTasks().values().iterator();
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // send the reduce resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for the reduce task to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish the reduce task
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ return resourceRequests;
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8757aa90/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index ca18bfe..48887c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1179,6 +1179,12 @@ public interface MRJobConfig {
public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
+ * Opt out of YARN oversubscription so that the job always waits for
+ * GUARANTEED resources available in the cluster.
+ */
+ String MR_OVERSUBSCRIPTION_OPT_OUT = "mapreduce.job.oversubscription-opt-out";
+ boolean DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT = false;
+ /**
* A comma-separated list of properties whose value will be redacted.
*/
String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8757aa90/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 9f33d65..de725a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2151,4 +2151,12 @@
</description>
</property>
+<property>
+ <description>
+ Opts out of YARN oversubscription so that the job always waits for GUARANTEED
+ resources available.
+ </description>
+ <name>mapreduce.job.oversubscription-opt-out</name>
+ <value>false</value>
+</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/24] hadoop git commit: YARN-6675. Add NM support to launch
opportunistic containers based on overallocation. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index 6b3ac67..f24df38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -66,8 +67,8 @@ public class TestContainerSchedulerRecovery {
@Mock private ContainerId containerId;
- @Mock private AllocationBasedResourceUtilizationTracker
- allocationBasedResourceUtilizationTracker;
+ @Mock private AllocationBasedResourceTracker
+ allocationBasedResourceTracker;
@InjectMocks private ContainerScheduler tempContainerScheduler =
new ContainerScheduler(context, dispatcher, metrics, 0);
@@ -85,12 +86,13 @@ public class TestContainerSchedulerRecovery {
MockitoAnnotations.initMocks(this);
spy = spy(tempContainerScheduler);
when(container.getContainerId()).thenReturn(containerId);
+ when(container.getResource()).thenReturn(Resource.newInstance(1024, 1));
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
when(containerId.getApplicationAttemptId().getApplicationId())
.thenReturn(appId);
when(containerId.getContainerId()).thenReturn(123L);
- doNothing().when(allocationBasedResourceUtilizationTracker)
- .addContainerResources(container);
+ doNothing().when(allocationBasedResourceTracker)
+ .containerLaunched(container);
}
@After public void tearDown() {
@@ -112,8 +114,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
@@ -132,8 +134,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, GUARANTEED,
@@ -152,8 +154,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(1, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
@@ -172,8 +174,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(1, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, GUARANTEED,
@@ -192,8 +194,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
@@ -212,8 +214,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(1, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(1))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, GUARANTEED,
@@ -232,8 +234,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
@@ -252,8 +254,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, GUARANTEED,
@@ -272,8 +274,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
@@ -292,8 +294,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as GUARANTEED but no executionType set,
@@ -311,8 +313,8 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
/*Test if a container is recovered as PAUSED but no executionType set,
@@ -330,7 +332,7 @@ public class TestContainerSchedulerRecovery {
assertEquals(0, spy.getNumQueuedGuaranteedContainers());
assertEquals(0, spy.getNumQueuedOpportunisticContainers());
assertEquals(0, spy.getNumRunningContainers());
- Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
- .addContainerResources(container);
+ Mockito.verify(allocationBasedResourceTracker, Mockito.times(0))
+ .containerLaunched(container);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/879a5e7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
new file mode 100644
index 0000000..384b116
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerWithOverAllocation.java
@@ -0,0 +1,1121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+/**
+ * Test ContainerScheduler behaviors when NM overallocation is turned on.
+ */
+public class TestContainerSchedulerWithOverAllocation
+ extends BaseContainerManagerTest {
+ private static final int NM_OPPORTUNISTIC_QUEUE_LIMIT = 3;
+ private static final int NM_CONTAINERS_VCORES = 4;
+ private static final int NM_CONTAINERS_MEMORY_MB = 2048;
+
+ static {
+ LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
+ }
+
+ public TestContainerSchedulerWithOverAllocation()
+ throws UnsupportedFileSystemException {
+ }
+
+ @Override
+ protected ContainerExecutor createContainerExecutor() {
+ DefaultContainerExecutor exec =
+ new LongRunningContainerSimulatingContainerExecutor();
+ exec.setConf(conf);
+ return exec;
+ }
+
+ @Override
+ protected ContainerManagerImpl createContainerManager(
+ DeletionService delSrvc) {
+ return new LongRunningContainerSimulatingContainersManager(
+ context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler, user);
+ }
+
+ @Override
+ public void setup() throws IOException {
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
+ NM_OPPORTUNISTIC_QUEUE_LIMIT);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_CPU_UTILIZATION_THRESHOLD,
+ 0.75f);
+ conf.setFloat(
+ YarnConfiguration.NM_OVERALLOCATION_MEMORY_UTILIZATION_THRESHOLD,
+ 0.75f);
+ super.setup();
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC container, which in aggregate do
+ * not exceed the capacity of the node. Both containers are expected to start
+ * running immediately.
+ */
+ @Test
+ public void testStartMultipleContainersWithoutOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ StartContainersRequest allRequests = StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() { {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ } }
+ );
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers whose utilization
+ * is very low relative to their resource request, resulting in a low node
+ * utilization. Then start another OPPORTUNISTIC containers which requests
+ * more than what's left unallocated on the node. Due to overallocation
+ * being turned on and node utilization being low, the second OPPORTUNISTIC
+ * container is also expected to be launched immediately.
+ */
+ @Test
+ public void testStartOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started immediately because there
+ // are (memory: 1024, vcore: 0.625) available based on over-allocation
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED and one OPPORTUNISTIC containers which utilizes most
+ * of the resources they requested, resulting in a high node utilization.
+ * Then start another OPPORTUNISTIC containers which requests more than what's
+ * left unallocated on the node. Because of the high resource utilization on
+ * the node, the projected utilization, if we were to start the second
+ * OPPORTUNISTIC container immediately, will go over the NM overallocation
+ * threshold, so the second OPPORTUNISTIC container is expected to be queued.
+ */
+ @Test
+ public void testQueueOppContainerWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(
+ containerManager, createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1500, 0, 1.0f/8));
+
+ // start a container that requests more than what's left unallocated
+ // 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+ // this container will not start immediately because there is not
+ // enough resource available at the moment either in terms of
+ // resources unallocated or in terms of the actual utilization
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+ }
+
+ /**
+ * Start two GUARANTEED containers which in aggregate takes up the whole node
+ * capacity, yet whose utilization is low relative to their resource request,
+ * resulting in a low node resource utilization. Then try to start another
+ * OPPORTUNISTIC containers. Because the resource utilization across the node
+ * is low and overallocation being turned on, the OPPORTUNISTIC container is
+ * expected to be launched immediately even though there is no resources left
+ * unallocated.
+ */
+ @Test
+ public void testStartOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the current containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(800, 0, 1.0f/8));
+
+ // start a container when there is no resources left unallocated.
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false))
+ ));
+
+ // this container is expected to be started because there is resources
+ // available because the actual utilization is very low
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ /**
+ * Start two GUARANTEED containers which in aggregate take up the whole node
+ * capacity and fully utilize the resources they requested. Then try to start
+ * four OPPORTUNISTIC containers of which three will be queued and one will be
+ * killed because of the max queue length is 3.
+ */
+ @Test
+ public void testQueueOppContainersWithFullUtilization() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers are fully utilizing their resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f/8));
+
+ // start more OPPORTUNISTIC containers than what the OPPORTUNISTIC container
+ // queue can hold when there is no unallocated resource left.
+ List<StartContainerRequest> moreContainerRequests =
+ new ArrayList<>(NM_OPPORTUNISTIC_QUEUE_LIMIT + 1);
+ for (int a = 0; a < NM_OPPORTUNISTIC_QUEUE_LIMIT + 1; a++) {
+ moreContainerRequests.add(
+ createStartContainerRequest(2 + a,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ containerManager.startContainers(
+ StartContainersRequest.newInstance(moreContainerRequests));
+
+ // All OPPORTUNISTIC containers but the last one should be queued.
+ // The last OPPORTUNISTIC container to launch should be killed.
+ BaseContainerManagerTest.waitForContainerState(
+ containerManager, createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT + 2),
+ ContainerState.COMPLETE);
+
+ HashMap<ContainerId, ContainerSubState> expectedContainerStatus =
+ new HashMap<>();
+ expectedContainerStatus.put(
+ createContainerId(0), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(1), ContainerSubState.RUNNING);
+ expectedContainerStatus.put(
+ createContainerId(NM_OPPORTUNISTIC_QUEUE_LIMIT),
+ ContainerSubState.DONE);
+ for (int i = 0; i < NM_OPPORTUNISTIC_QUEUE_LIMIT; i++) {
+ expectedContainerStatus.put(
+ createContainerId(i + 2), ContainerSubState.SCHEDULED);
+ }
+ verifyContainerStatuses(expectedContainerStatus);
+ }
+
+ /**
+ * Start two GUARANTEED containers that together does not take up the
+ * whole node. Then try to start one OPPORTUNISTIC container that will
+ * fit into the remaining unallocated space on the node.
+ * The OPPORTUNISTIC container is expected to start even though the
+ * current node utilization is above the NM overallocation threshold,
+ * because it's always safe to launch containers as long as the node
+ * has not been fully allocated.
+ */
+ @Test
+ public void testStartOppContainerWithHighUtilizationNoOverallocation()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1200, 1), true));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(400, 1), true));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is above the over-allocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1600, 0, 1.0f/2));
+
+ // start a container that can just fit in the remaining unallocated space
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(400, 1), false))
+ ));
+
+ // the OPPORTUNISTIC container can be safely launched even though
+ // the container utilization is above the NM overallocation threshold
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers first whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization. Then
+ * try to start a GUARANTEED container which requests more than what's left
+ * unallocated on the node. Because the node utilization is low and NM
+ * overallocation is turned on, the GUARANTEED container is expected to be
+ * started immediately without killing any running OPPORTUNISTIC containers.
+ */
+ @Test
+ public void testKillNoOppContainersWithPartialOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers.
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers whose utilization will be high relative
+ * to the resources they requested, resulting in a high node utilization.
+ * Then try to start a GUARANTEED container which requests more than what's
+ * left unallocated on the node. Because the node is under high utilization,
+ * the second OPPORTUNISTIC container is expected to be killed in order to
+ * make room for the GUARANTEED container.
+ */
+ @Test
+ public void testKillOppContainersWithPartialOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(824, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+
+ // the containers utilization is very high
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1800, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node 512 + 1024 + 824 > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ // the last launched OPPORTUNISTIC container is expected to be killed
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.DONE);
+
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ } else {
+ Assert.assertEquals(status.getContainerId() + " is not RUNNING",
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+
+ /**
+ * Start three OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node, yet whose utilization is low relative
+ * to the resources they requested, resulting in a low node utilization.
+ * Then try to start a GUARANTEED container. Even though the node has
+ * nothing left unallocated, it is expected to start immediately
+ * without killing any running OPPORTUNISTIC containers because the node
+ * utilization is very low and overallocation is turned on.
+ */
+ @Test
+ public void testKillNoOppContainersWithOverallocationLowUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All three GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the containers utilization is low
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1024, 0, 1.0f/8));
+
+ // start a GUARANTEED container that requests more than what's left
+ // unallocated on the node: (512 + 1024 + 824) > 2048
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), true))
+ ));
+
+ // the GUARANTEED container is expected be launched immediately without
+ // killing any OPPORTUNISTIC containers
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start four OPPORTUNISTIC containers which in aggregates exceeds the
+ * capacity of the node. The real resource utilization of the first two
+ * OPPORTUNISTIC containers are high whereas that of the latter two are
+ * almost zero. Then try to start a GUARANTEED container. The GUARANTEED
+ * container will eventually start running after preempting the third
+ * and fourth OPPORTUNISTIC container (which releases no resources) and
+ * then the second OPPORTUNISTIC container.
+ */
+ @Test
+ public void
+ testKillOppContainersConservativelyWithOverallocationHighUtilization()
+ throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(1024, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(1024, 1), false));
+ }
+ }
+ ));
+ // All four GUARANTEED containers are all expected to start
+ // because the containers utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+
+ // the containers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start a GUARANTEED container when there's nothing left unallocated
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(4,
+ BuilderUtils.newResource(1024, 1), true))
+ ));
+
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+ GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.
+ newInstance(new ArrayList<ContainerId>() {
+ {
+ add(createContainerId(0));
+ add(createContainerId(1));
+ add(createContainerId(2));
+ add(createContainerId(3));
+ add(createContainerId(4));
+ }
+ });
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0)) ||
+ status.getContainerId().equals(createContainerId(4))) {
+ Assert.assertEquals(
+ ContainerSubState.RUNNING, status.getContainerSubState());
+ } else {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Killed to make room for Guaranteed Container"));
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+ }
+
+ /**
+ * Start two OPPORTUNISTIC containers followed by one GUARANTEED container,
+ * which in aggregate exceeds the capacity of the node. The first two
+ * OPPORTUNISTIC containers use almost no resources whereas the GUARANTEED
+ * one utilizes nearly all of its resource requested. Then try to start two
+ * more OPPORTUNISTIC containers. The two OPPORTUNISTIC containers are
+ * expected to be queued immediately. Upon the completion of the
+ * resource-usage-heavy GUARANTEED container, both OPPORTUNISTIC containers
+ * are expected to start.
+ */
+ @Test
+ public void testStartOppContainersUponContainerCompletion() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(0,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(1024, 1), true));
+ }
+ }
+ ));
+
+ // All three containers are all expected to start immediately
+ // because the node utilization is low (0 at the point)
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ // the contianers utilization is at the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(3,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(4,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ // the two new OPPORTUNISTIC containers are expected to be queued
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(3), ContainerSubState.SCHEDULED);
+ put(createContainerId(4), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+ allowContainerToSucceed(2);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.DONE);
+
+ // the two OPPORTUNISTIC containers are expected to start together
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(3), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(4), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.DONE);
+ put(createContainerId(3), ContainerSubState.RUNNING);
+ put(createContainerId(4), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+ /**
+ * Start one GUARANTEED container that consumes all the resources on the
+ * node and keeps running, followed by two OPPORTUNISTIC containers that
+ * will be queued forever because there is no containers starting or
+ * finishing. Then try to start OPPORTUNISTIC containers out of band.
+ */
+ @Test
+ public void testStartOpportunisticContainersOutOfBand() throws Exception {
+ containerManager.start();
+
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ Collections.singletonList(
+ createStartContainerRequest(0,
+ BuilderUtils.newResource(2048, 4), true))));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(0), ContainerSubState.RUNNING);
+
+ // the container is fully utilizing its resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(2048, 0, 1.0f));
+
+ // send two OPPORTUNISTIC container requests that are expected to be queued
+ containerManager.startContainers(StartContainersRequest.newInstance(
+ new ArrayList<StartContainerRequest>() {
+ {
+ add(createStartContainerRequest(1,
+ BuilderUtils.newResource(512, 1), false));
+ add(createStartContainerRequest(2,
+ BuilderUtils.newResource(512, 1), false));
+ }
+ }
+ ));
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.SCHEDULED);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.SCHEDULED);
+
+ // the containers utilization dropped to the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(1536, 0, 1.0f/2));
+
+ // try to start containers out of band.
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // no containers in queue are expected to be launched because the
+ // containers utilization is not below the over-allocation threshold
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.SCHEDULED);
+ put(createContainerId(2), ContainerSubState.SCHEDULED);
+ }
+ });
+
+ // the GUARANTEED container is completed releasing resources
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(100, 0, 1.0f/5));
+
+ // the containers utilization dropped way below the overallocation threshold
+ setContainerResourceUtilization(
+ ResourceUtilization.newInstance(512, 0, 1.0f/8));
+
+ ((LongRunningContainerSimulatingContainersManager)containerManager)
+ .startContainersOutOfBandUponLowUtilization();
+
+ // the two OPPORTUNISTIC containers are expected to be launched
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(1), ContainerSubState.RUNNING);
+ BaseContainerManagerTest.waitForContainerSubState(containerManager,
+ createContainerId(2), ContainerSubState.RUNNING);
+
+ verifyContainerStatuses(new HashMap<ContainerId, ContainerSubState>() {
+ {
+ put(createContainerId(0), ContainerSubState.RUNNING);
+ put(createContainerId(1), ContainerSubState.RUNNING);
+ put(createContainerId(2), ContainerSubState.RUNNING);
+ }
+ });
+ }
+
+
+ private void setContainerResourceUtilization(ResourceUtilization usage) {
+ ((ContainerMonitorForOverallocationTest)
+ containerManager.getContainersMonitor())
+ .setContainerResourceUsage(usage);
+ }
+
+ private void allowContainerToSucceed(int containerId) {
+ ((LongRunningContainerSimulatingContainerExecutor) this.exec)
+ .containerSucceeded(createContainerId(containerId));
+ }
+
+
+ protected StartContainerRequest createStartContainerRequest(int containerId,
+ Resource resource, boolean isGuaranteed) throws IOException {
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ExecutionType executionType = isGuaranteed ? ExecutionType.GUARANTEED :
+ ExecutionType.OPPORTUNISTIC;
+ Token containerToken = createContainerToken(
+ createContainerId(containerId),
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user, resource,
+ context.getContainerTokenSecretManager(),
+ null, executionType);
+
+ return StartContainerRequest.newInstance(
+ containerLaunchContext, containerToken);
+ }
+
+ protected void verifyContainerStatuses(
+ Map<ContainerId, ContainerSubState> expected)
+ throws IOException, YarnException {
+ List<ContainerId> statList = new ArrayList<>(expected.keySet());
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+
+ for (ContainerStatus status : containerStatuses) {
+ ContainerId containerId = status.getContainerId();
+ Assert.assertEquals(containerId + " is in unexpected state",
+ expected.get(containerId), status.getContainerSubState());
+ }
+ }
+
+ /**
+ * A container manager that sends a dummy container pid while it's cleaning
+ * up running containers. Used along with
+ * LongRunningContainerSimulatingContainerExecutor to simulate long running
+ * container processes for testing purposes.
+ */
+ private static class LongRunningContainerSimulatingContainersManager
+ extends ContainerManagerImpl {
+
+ private final String user;
+
+ LongRunningContainerSimulatingContainersManager(
+ Context context, ContainerExecutor exec,
+ DeletionService deletionContext,
+ NodeStatusUpdater nodeStatusUpdater,
+ NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler, String user) {
+ super(context, exec, deletionContext,
+ nodeStatusUpdater, metrics, dirsHandler);
+ this.user = user;
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ /**
+ * Create a container launcher that signals container processes
+ * with a dummy pid. The container processes are simulated in
+ * LongRunningContainerSimulatingContainerExecutor which does
+ * not write a pid file on behalf of containers to launch, so
+ * the pid does not matter.
+ */
+ @Override
+ protected ContainersLauncher createContainersLauncher(
+ Context context, ContainerExecutor exec) {
+ ContainerManagerImpl containerManager = this;
+ return new ContainersLauncher(context, dispatcher, exec, dirsHandler,
+ this) {
+ @Override
+ protected ContainerLaunch createContainerLaunch(
+ Application app, Container container) {
+ return new ContainerLaunch(context, getConfig(), dispatcher,
+ exec, app, container, dirsHandler, containerManager) {
+ @Override
+ protected String getContainerPid(Path pidFilePath)
+ throws Exception {
+ return "123";
+ }
+
+ };
+ }
+ };
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(
+ ContainerExecutor exec) {
+ return new ContainerMonitorForOverallocationTest(exec,
+ dispatcher, context);
+ }
+
+ public void startContainersOutOfBandUponLowUtilization() {
+ ((ContainerMonitorForOverallocationTest) getContainersMonitor())
+ .attemptToStartContainersUponLowUtilization();
+ }
+ }
+
+ /**
+ * A container executor that simulates long running container processes
+ * by having container launch threads sleep infinitely until it's given
+ * a signal to finish with either a success or failure exit code.
+ */
+ private static class LongRunningContainerSimulatingContainerExecutor
+ extends DefaultContainerExecutor {
+ private ConcurrentHashMap<ContainerId, ContainerFinishLatch> containers =
+ new ConcurrentHashMap<>();
+
+ public void containerSucceeded(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toSucceed();
+ }
+ }
+
+ public void containerFailed(ContainerId containerId) {
+ ContainerFinishLatch containerFinishLatch = containers.get(containerId);
+ if (containerFinishLatch != null) {
+ containerFinishLatch.toFail();
+ }
+ }
+
+ /**
+ * Simulate long running container processes by having container launcher
+ * threads wait infinitely for a signal to finish.
+ */
+ @Override
+ public int launchContainer(ContainerStartContext ctx)
+ throws IOException, ConfigurationException {
+ ContainerId container = ctx.getContainer().getContainerId();
+ containers.putIfAbsent(container, new ContainerFinishLatch(container));
+
+ // simulate a long running container process by having the
+ // container launch thread sleep forever until it's given a
+ // signal to finish with a exit code.
+ while (!containers.get(container).toProceed) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ return -1;
+ }
+ }
+
+ return containers.get(container).getContainerExitCode();
+ }
+
+ /**
+ * Override signalContainer() so that simulated container processes
+ * are properly cleaned up.
+ */
+ @Override
+ public boolean signalContainer(ContainerSignalContext ctx)
+ throws IOException {
+ containerSucceeded(ctx.getContainer().getContainerId());
+ return true;
+ }
+
+ /**
+ * A signal that container launch threads wait for before exiting
+ * in order to simulate long running container processes.
+ */
+ private static final class ContainerFinishLatch {
+ volatile boolean toProceed;
+ int exitCode;
+ ContainerId container;
+
+ ContainerFinishLatch(ContainerId containerId) {
+ exitCode = 0;
+ toProceed = false;
+ container = containerId;
+ }
+
+ void toSucceed() {
+ exitCode = 0;
+ toProceed = true;
+ }
+
+ void toFail() {
+ exitCode = -101;
+ toProceed = true;
+ }
+
+ int getContainerExitCode() {
+ // read barrier of toProceed to make sure the exit code is not stale
+ if (toProceed) {
+ LOG.debug(container + " finished with exit code: " + exitCode);
+ }
+ return exitCode;
+ }
+ }
+ }
+
+ /**
+ * A test implementation of container monitor that allows control of
+ * current resource utilization.
+ */
+ private static class ContainerMonitorForOverallocationTest
+ extends ContainersMonitorImpl {
+
+ private ResourceUtilization containerResourceUsage =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
+ ContainerMonitorForOverallocationTest(ContainerExecutor exec,
+ AsyncDispatcher dispatcher, Context context) {
+ super(exec, dispatcher, context);
+ }
+
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return NM_CONTAINERS_MEMORY_MB * 1024 * 1024L;
+ }
+
+ @Override
+ public long getVmemAllocatedForContainers() {
+ float pmemRatio = getConfig().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ return (long) (pmemRatio * getPmemAllocatedForContainers());
+ }
+
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return NM_CONTAINERS_VCORES;
+ }
+
+ @Override
+ public ContainersResourceUtilization getContainersUtilization(
+ boolean latest) {
+ return new ContainersMonitor.ContainersResourceUtilization(
+ containerResourceUsage, System.currentTimeMillis());
+ }
+
+ public void setContainerResourceUsage(
+ ResourceUtilization containerResourceUsage) {
+ this.containerResourceUsage = containerResourceUsage;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/24] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public long getMemorySeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemorySeconds();
+ return getGuaranteedMemorySeconds();
}
@Override
+ @Deprecated
public long getVcoreSeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVcoreSeconds();
+ return getGuaranteedVcoreSeconds();
}
@Override
+ @Deprecated
public void setMemorySeconds(long memorySeconds) {
- maybeInitBuilder();
- builder.setMemorySeconds(memorySeconds);
+ setGuaranteedMemorySeconds(memorySeconds);
}
@Override
+ @Deprecated
public void setVcoreSeconds(long vcoreSeconds) {
+ setGuaranteedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public long getGuaranteedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public void setGuaranteedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public long getGuaranteedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcoreSeconds);
}
@Override
+ public long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
public long getPreemptedMemorySeconds() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ @Deprecated
+ public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ setGuaranteedResourceSecondsMap(resourceSecondsMap);
+ }
+
+ @Override
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
- p.getApplicationResourceUsageMapList());
- return this.resourceSecondsMap;
+ this.guaranteedResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationResourceUsageMapList());
+ return this.guaranteedResourceSecondsMap;
}
@Override
- public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ public void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
+
+ @Override
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
+ maybeInitBuilder();
+ builder.clearApplicationOpportunisticResourceUsageMap();
+ this.opportunisticResourceSecondsMap = resourceSecondsMap;
+ if (resourceSecondsMap != null) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 42e2bcf..7eb2379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -456,6 +456,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
+ LOG.info(applicationId.toString() + " started at " + this.startTime);
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
@@ -775,10 +776,12 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
- appUsageReport
- .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+ appUsageReport.setGuaranteedResourceSecondsMap(
+ rmAppMetrics.getGuaranteedResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
+ appUsageReport.setOpportunisticResourceSecondsMap(
+ rmAppMetrics.getOpportunisticResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@@ -1705,8 +1708,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+ Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+ Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
this.readLock.lock();
try {
for (RMAppAttempt attempt : attempts.values()) {
@@ -1722,20 +1726,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ guaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap(),
+ opportunsiticResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ attemptMetrics.getPreemptedResourceSecondsMap(),
+ preemptedSecondsMap);
}
}
} finally {
@@ -1743,7 +1742,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
- numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+ numAMContainerPreempted, guaranteedResourceSecondsMap,
+ preemptedSecondsMap, opportunsiticResourceSecondsMap);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
- private final Map<String, Long> resourceSecondsMap;
+ private final Map<String, Long> guaranteedResourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
}
public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
return numAMContainersPreempted;
}
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
}
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
public long getPreemptedMemorySeconds() {
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceSecondsMap() {
- return resourceSecondsMap;
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ return guaranteedResourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
@Private
public class AggregateAppResourceUsage {
- private Map<String, Long> resourceSecondsMap = new HashMap<>();
+ private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
- public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap.putAll(resourceSecondsMap);
+ public AggregateAppResourceUsage(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap.putAll(
+ guaranteedResourceSecondsMap);
+ this.opportunisticResourceSecondsMap.putAll(
+ opportunisticResourceSecondsMap);
}
/**
- * @return the memorySeconds
+ * @return the guaranteed memory seconds
*/
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
- * @return the vcoreSeconds
+ * @return the guaranteed vcore seconds
*/
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceUsageSecondsMap() {
- return resourceSecondsMap;
+ /**
+ * @return the opportunistic memory seconds
+ */
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+
+ /**
+ * @return the opportunistic vcore seconds
+ */
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3ec9c49..b0fa02f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -938,15 +938,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock.lock();
try {
ApplicationResourceUsageReport report =
- scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+ scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
- report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+ report.setGuaranteedResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
+ report.setOpportunisticResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@@ -983,10 +986,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
- this.attemptMetrics
- .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+ attemptState.getGuaranteedResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+ attemptState.getOpportunisticResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1406,9 +1411,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
- getFinishTime(), resUsage.getResourceUsageSecondsMap(),
- this.attemptMetrics.getPreemptedResourceSecondsMap());
+ stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+ exitStatus, getFinishTime(),
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ this.attemptMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
@@ -1837,8 +1844,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
newTrackingUrl, appAttempt.getDiagnostics(), null,
ContainerExitStatus.INVALID, appAttempt.getFinishTime(),
appAttempt.attemptMetrics.getAggregateAppResourceUsage()
- .getResourceUsageSecondsMap(),
- appAttempt.attemptMetrics.getPreemptedResourceSecondsMap());
+ .getGuaranteedResourceUsageSecondsMap(),
+ appAttempt.attemptMetrics.getPreemptedResourceSecondsMap(),
+ appAttempt.attemptMetrics.getAggregateAppResourceUsage()
+ .getOpportunisticResourceSecondsMap());
appAttempt.rmContext.getStateStore()
.updateApplicationAttemptState(attemptState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 43e6e4d..59fed88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -54,8 +54,11 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
- private Map<String, AtomicLong> resourceUsageMap = new ConcurrentHashMap<>();
- private Map<String, AtomicLong> preemptedResourceMap = new ConcurrentHashMap<>();
+ private Map<String, AtomicLong> guaranteedResourceUsageMap
+ = new HashMap<>();
+ private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+ private Map<String, AtomicLong> opportunisticResourceUsageMap
+ = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@@ -130,8 +133,10 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
- Map<String, Long> resourcesUsed =
- convertAtomicLongMaptoLongMap(resourceUsageMap);
+ Map<String, Long> guaranteedResourcesUsed =
+ convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+ Map<String, Long> opportunisticResourcesUsed =
+ convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -140,27 +145,25 @@ public class RMAppAttemptMetrics {
if (currentAttempt != null
&& currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
- rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+ rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
if (appResUsageReport != null) {
- Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
- for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- Long value = resourcesUsed.get(entry.getKey());
- if (value != null) {
- value += entry.getValue();
- } else {
- value = entry.getValue();
- }
- resourcesUsed.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getGuaranteedResourceSecondsMap(),
+ guaranteedResourcesUsed);
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getOpportunisticResourceSecondsMap(),
+ opportunisticResourcesUsed);
}
}
}
- return new AggregateAppResourceUsage(resourcesUsed);
+
+ return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+ opportunisticResourcesUsed);
}
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
- updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+ updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -168,9 +171,14 @@ public class RMAppAttemptMetrics {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
- public void updateAggregateAppResourceUsage(
+ public void updateAggregateAppGuaranteedResourceUsage(
Map<String, Long> resourceSecondsMap) {
- updateUsageMap(resourceSecondsMap, resourceUsageMap);
+ updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+ }
+
+ public void updateAggregateAppOpportunisticResourceUsage(
+ Map<String, Long> resourceSecondsMap) {
+ updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
@@ -200,10 +208,9 @@ public class RMAppAttemptMetrics {
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
-
}
resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.set(entry.getValue());
+ resourceUsed.addAndGet(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+ private final Map<String, Long> guaranteedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
+
+ public ContainerResourceUsageReport(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 8f751b0..4897312 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -87,7 +87,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
ContainerState getContainerState();
ContainerReport createContainerReport();
-
+
+ ContainerResourceUsageReport getResourceUsageReport();
+
boolean isAMContainer();
ContainerRequest getContainerRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 945e7cb..ccd02ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +179,7 @@ public class RMContainerImpl implements RMContainer {
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
private long creationTime;
- private long finishTime;
+ private long finishTime = -1L;
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private ContainerRequest containerRequestForRecovery;
@@ -191,6 +194,10 @@ public class RMContainerImpl implements RMContainer {
private volatile Set<String> allocationTags = null;
+ // This is updated whenever the container instance is updated in
+ // the case of container increase/decrease/promotion/demotion
+ private long lastContainerUpdateTime;
+
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -232,6 +239,7 @@ public class RMContainerImpl implements RMContainer {
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
+ this.lastContainerUpdateTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -272,9 +280,87 @@ public class RMContainerImpl implements RMContainer {
}
public void setContainer(Container container) {
+ // containers are updated by resetting the underlying Container instance,
+ // always update RMContainer resource usage before the update.
+ updateAppAttemptMetrics(true);
this.container = container;
}
+ /**
+ * Collect the resource usage information of the current underlying Container
+ * instance and update its RMAppAttemptMetrics. This is called whenever the
+ * RMContainer is updated (by updating its underlying container) or finishes.
+ * @param containerUpdated if it is called upon container update event
+ */
+ private void updateAppAttemptMetrics(boolean containerUpdated) {
+ RMAppAttempt rmAttempt = rmContext.getRMApps()
+ .get(getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+
+ if (rmAttempt != null) {
+ // collect resource usage information of the current Container instance
+ ContainerResourceUsageReport resourceUsage =
+ getCurrentContainerResourceUsage(containerUpdated);
+
+ // If this is a preempted container, update preemption metrics
+ if (finishedStatus != null &&
+ ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics()
+ .updatePreemptionInfo(container.getResource(), this);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getOpportunisticResourceSecondsMap());
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppGuaranteedResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ }
+
+ /**
+ * Get resource usage of the current underlying Container instance.
+ * @param containerUpdated if this is called upon container update
+ * @return resource usage of the current <code>container</code>
+ */
+ private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+ boolean containerUpdated) {
+ ContainerResourceUsageReport report;
+
+ // A container generates usage until it finishes which is indicated
+ // by a positive finish timestamp that is set once it finishes
+ final long currentTimeMillis =
+ finishTime < 0 ? System.currentTimeMillis() : finishTime;
+ final long usedSeconds = (currentTimeMillis -
+ lastContainerUpdateTime) / 1000;
+ Resource resource = container.getResource();
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ guaranteedResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+ Collections.emptyMap());
+ } else {
+ Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ opportunisticResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(Collections.emptyMap(),
+ opportunisticResourceSeconds);
+ }
+
+ if (containerUpdated) {
+ lastContainerUpdateTime = currentTimeMillis;
+ }
+ return report;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
@@ -704,7 +790,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateAttemptMetrics(container);
+ container.updateAppAttemptMetrics(false);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -721,28 +807,6 @@ public class RMContainerImpl implements RMContainer {
container, container.finishTime);
}
}
-
- private static void updateAttemptMetrics(RMContainerImpl container) {
- Resource resource = container.getContainer().getResource();
- RMApp app = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId());
- if (app != null) {
- RMAppAttempt rmAttempt = app.getCurrentAppAttempt();
- if (rmAttempt != null) {
- long usedMillis = container.finishTime - container.creationTime;
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregateAppResourceUsage(resource, usedMillis);
- // If this is a preempted container, update preemption metrics
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
- }
- }
- }
- }
}
private static boolean shouldPublishNonAMContainerEventstoATS(
@@ -804,6 +868,11 @@ public class RMContainerImpl implements RMContainer {
}
@Override
+ public ContainerResourceUsageReport getResourceUsageReport() {
+ return getCurrentContainerResourceUsage(false);
+ }
+
+ @Override
public String getNodeHttpAddress() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 278e474..a8f5093 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -417,7 +417,7 @@ public abstract class AbstractYarnScheduler
}
@Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
+ public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId) {
SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
if (attempt == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f9df2b8..25a9415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -61,12 +61,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -108,7 +108,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
- private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> lastGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ private Map<String, Long> lastOpportunisticResourceSecondsMap =
+ new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -1083,34 +1086,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+ new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
- long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
- Resource resource = rmContainer.getContainer().getResource();
- for (ResourceInformation entry : resource.getResources()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
- value += entry.getValue() * usedMillis
- / DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getName(), value);
- }
+ ContainerResourceUsageReport containerResourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ aggregatedGuaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+ aggregatedOpportunisticResourceSecondsMap);
}
-
+ lastGuaranteedResourceSecondsMap =
+ aggregatedGuaranteedResourceSecondsMap;
+ lastOpportunisticResourceSecondsMap =
+ aggregatedOpportunisticResourceSecondsMap;
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
- lastResourceSecondsMap = resourceSecondsMap;
}
- return new AggregateAppResourceUsage(lastResourceSecondsMap);
+ return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+ lastOpportunisticResourceSecondsMap);
}
+ /**
+ * Get the resources that are actively being used by the app attempt.
+ * @return active resource usage
+ */
public ApplicationResourceUsageReport getResourceUsageReport() {
try {
writeLock.lock();
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
- Resource usedResourceClone = Resources.clone(
+ Resource guaranteedResourceUsedClone = Resources.clone(
attemptResourceUsage.getAllUsed());
+ Resource opportunisticResourceUsedClone = Resources.clone(
+ attemptOpportunisticResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
+ Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+ Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
@@ -1125,18 +1141,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
float queueCapacityPerc = queue.getQueueInfo(false, false)
.getCapacity();
if (queueCapacityPerc != 0) {
- queueUsagePerc = calc.divide(cluster, usedResourceClone,
+ queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc =
- calc.divide(cluster, usedResourceClone, cluster) * 100;
+ calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
}
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
- usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
- clusterUsagePerc, preemptedResourceSecondsMaps);
+ guaranteedResourceUsedClone, reservedResourceClone,
+ neededResource,
+ runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+ queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+ opportunisticResourceUsedClone,
+ runningResourceUsage.getOpportunisticResourceSecondsMap());
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 0f7a5b5..2a52389 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -172,7 +172,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
*/
@LimitedPrivate("yarn")
@Evolving
- ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index e6a71f6..4bd22584 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1111,7 +1111,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.getEffectiveCapacity(getAppAMNodePartitionName());
if (!effCap.equals(Resources.none())) {
queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(), effCap) * 100;
+ report.getGuaranteedResourcesUsed(), effCap) * 100;
}
report.setQueueUsagePercentage(queueUsagePerc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0effd42..7b32a9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedGuaranteedResource(containerResource);
+ if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
+ } else {
+ this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+ }
+
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 14ad277..3a5c41e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -96,6 +96,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".runningcontainer", "Running Containers").
th(".allocatedCpu", "Allocated CPU VCores").
th(".allocatedMemory", "Allocated Memory MB").
+ th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU Vcores").
+ th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB").
th(".reservedCpu", "Reserved CPU VCores").
th(".reservedMemory", "Reserved Memory MB").
th(".progress", "Progress").
@@ -146,6 +150,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
.append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
.valueOf(appInfo.getAllocatedMB())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
.append(appInfo.getReservedVCores() == -1 ? "N/A" : String
.valueOf(appInfo.getReservedVCores())).append("\",\"")
.append(appInfo.getReservedMB() == -1 ? "N/A" : String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index d260400..d5e8d3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
- StringHelper
- .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+ .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getGuaranteedResourceSecondsMap()))
+ .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getOpportunisticResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index b1c0cd9..b937c41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -70,6 +70,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU VCores")
+ .th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -157,6 +161,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 63b6fe0..14795d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -97,6 +97,8 @@ public class AppInfo {
private String masterNodeId;
private long allocatedMB;
private long allocatedVCores;
+ private long allocatedOpportunisticMB;
+ private long allocatedOpportunisticVCores;
private long reservedMB;
private long reservedVCores;
private int runningContainers;
@@ -105,6 +107,9 @@ public class AppInfo {
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
+ private long opportunisticMemorySeconds;
+ private long opportunisticVcoreSeconds;
+ private Map<String, Long> opportunisticResourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -205,10 +210,17 @@ public class AppInfo {
ApplicationResourceUsageReport resourceReport =
attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
- Resource usedResources = resourceReport.getUsedResources();
+ Resource guaranteedResourcesUsed =
+ resourceReport.getGuaranteedResourcesUsed();
+ Resource opportunisticResourceUsed =
+ resourceReport.getOpportunisticResourcesUsed();
Resource reservedResources = resourceReport.getReservedResources();
- allocatedMB = usedResources.getMemorySize();
- allocatedVCores = usedResources.getVirtualCores();
+ allocatedMB = guaranteedResourcesUsed.getMemorySize();
+ allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+ allocatedOpportunisticMB =
+ opportunisticResourceUsed.getMemorySize();
+ allocatedOpportunisticVCores =
+ opportunisticResourceUsed.getVirtualCores();
reservedMB = reservedResources.getMemorySize();
reservedVCores = reservedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
@@ -251,9 +263,13 @@ public class AppInfo {
numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
- memorySeconds = appMetrics.getMemorySeconds();
- vcoreSeconds = appMetrics.getVcoreSeconds();
- resourceSecondsMap = appMetrics.getResourceSecondsMap();
+ memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+ vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+ resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+ opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+ opportunisticResourceSecondsMap =
+ appMetrics.getOpportunisticResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -470,6 +486,13 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public long getAllocatedOpportunisticMB() {
+ return this.allocatedOpportunisticMB;
+ }
+
+ public long getAllocatedOpportunisticVCores() {
+ return this.allocatedOpportunisticVCores;
+ }
public long getReservedMB() {
return this.reservedMB;
}
@@ -494,18 +517,30 @@ public class AppInfo {
return numAMContainerPreempted;
}
- public long getMemorySeconds() {
+ public long getGuaranteedMemorySeconds() {
return memorySeconds;
}
- public long getVcoreSeconds() {
+ public long getGuaranteedVcoreSeconds() {
return vcoreSeconds;
}
- public Map<String, Long> getResourceSecondsMap() {
+ public long getOpportunisticMemorySeconds() {
+ return opportunisticMemorySeconds;
+ }
+
+ public long getOpportunisticVcoreSeconds() {
+ return opportunisticVcoreSeconds;
+ }
+
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
return resourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -598,6 +633,15 @@ public class AppInfo {
this.allocatedVCores = allocatedVCores;
}
+ public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+ this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+ }
+
+ public void setAllocatedOpportunisticVCores(
+ long allocatedOpportunisticVCores) {
+ this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+ }
+
public void setReservedMB(long reservedMB) {
this.reservedMB = reservedMB;
}
@@ -610,12 +654,20 @@ public class AppInfo {
this.runningContainers = runningContainers;
}
- public void setMemorySeconds(long memorySeconds) {
- this.memorySeconds = memorySeconds;
+ public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+ this.memorySeconds = guaranteedMemorySeconds;
+ }
+
+ public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+ this.vcoreSeconds = guaranteedVcoreSeconds;
+ }
+
+ public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+ this.opportunisticMemorySeconds = oppMemorySeconds;
}
- public void setVcoreSeconds(long vcoreSeconds) {
- this.vcoreSeconds = vcoreSeconds;
+ public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+ this.opportunisticVcoreSeconds = oppVcoreSeconds;
}
public void setAppId(String appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 35c77ab..3763339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -90,6 +90,7 @@ message ApplicationAttemptStateDataProto {
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8617dc8..febea7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1179,12 +1179,20 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, resourceSecondsMap, new HashMap<>());
+ 10, 1, guaranteedResourceSecondsMap,
+ new HashMap<>(), opportunisticResourceSecondsMap);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1195,16 +1203,19 @@ public class TestAppManager{
Assert.assertFalse(msg.contains("\r"));
String escaped = "\\n\\n\\r\\r";
- assertTrue(msg.contains("Multiline" + escaped +"AppName"));
- assertTrue(msg.contains("Multiline" + escaped +"UserName"));
- assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- assertTrue(msg.contains("submitTime=1000"));
- assertTrue(msg.contains("memorySeconds=16384"));
- assertTrue(msg.contains("vcoreSeconds=64"));
- assertTrue(msg.contains("preemptedAMContainers=1"));
- assertTrue(msg.contains("preemptedNonAMContainers=10"));
- assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
- assertTrue(msg.contains("applicationType=MAPREDUCE"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+ Assert.assertTrue(msg.contains("submitTime=1000"));
+ Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(
+ msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
- -1, usageReport.getUsedResources().getMemorySize());
+ -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemorySize());
+ Assert.assertEquals("Enemy should not see app used opportunistic resources",
+ -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 61048a5..a2ca4b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -385,8 +385,8 @@ public class TestClientRMService {
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
- Assert.assertEquals(10, usageReport.getMemorySeconds());
- Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
@@ -1421,26 +1421,28 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
- RMAppImpl app =
- spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+ ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ Resource.newInstance(1024, 1), 1);
+ RMAppImpl rmApp =
+ new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- Collections.singletonList(BuilderUtils.newResourceRequest(
- RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1))){
- @Override
- public ApplicationReport createAndGetApplicationReport(
- String clientUserName, boolean allowAccess) {
- ApplicationReport report = super.createAndGetApplicationReport(
- clientUserName, allowAccess);
- ApplicationResourceUsageReport usageReport =
- report.getApplicationResourceUsageReport();
- usageReport.setMemorySeconds(memorySeconds);
- usageReport.setVcoreSeconds(vcoreSeconds);
- report.setApplicationResourceUsageReport(usageReport);
- return report;
- }
- });
+ Collections.singletonList(resourceRequest)) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setGuaranteedMemorySeconds(memorySeconds);
+ usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ };
+ RMAppImpl app = spy(rmApp);
app.getAMResourceRequests().get(0)
.setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1496,7 +1498,8 @@ public class TestClientRMService {
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+ .thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/24] hadoop git commit: YARN-6750. Add a configuration to cap how
much a NM can be overallocated. Contributed by Haibo Chen.
Posted by ha...@apache.org.
YARN-6750. Add a configuration to cap how much a NM can be overallocated. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d028d77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d028d77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d028d77
Branch: refs/heads/YARN-1011
Commit: 6d028d77ca7c4650065251ea7e175fefe055eafa
Parents: d106fdb
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:17:56 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:06:14 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 10 ++
.../scheduler/AbstractYarnScheduler.java | 4 +
.../scheduler/SchedulerNode.java | 42 ++++++--
.../scheduler/fair/FSSchedulerNode.java | 5 +
.../scheduler/fair/FairScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 107 +++++++++++++++++++
7 files changed, 168 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 71651a6..0326880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -398,6 +398,11 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /** The global max overallocation per node in terms of their capacity. */
+ public static final String PER_NODE_MAX_OVERALLOCATION_RATIO =
+ RM_PREFIX + "overallocation.per-node-max-ratio";
+ public static final float DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO = 4.0f;
+
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b45fe06..bc9279e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3342,6 +3342,16 @@
<property>
<description>
+ The maximum amount of resources, specified as a ratio to node capacity,
+ that can be allocated to opportunistic containers on any given node in
+ the cluster.
+ </description>
+ <name>yarn.resourcemanager.overallocation.per-node-max-ratio</name>
+ <value>4.0</value>
+ </property>
+
+ <property>
+ <description>
Frequency for computing least loaded NMs.
</description>
<name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 6b1fdcb..278e474 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -119,6 +119,7 @@ public abstract class AbstractYarnScheduler
new ClusterNodeTracker<>();
protected Resource minimumAllocation;
+ protected float maxOverAllocationRatioPerNode;
protected volatile RMContext rmContext;
@@ -201,6 +202,9 @@ public abstract class AbstractYarnScheduler
nodeTracker.setConfiguredMaxAllocationWaitTime(
configuredMaximumAllocationWaitTime);
maxClusterLevelAppPriority = getMaxPriorityFromConf(conf);
+ maxOverAllocationRatioPerNode = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
createReleaseCache();
autoUpdateContainers =
conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index ff3640c..3e16dc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -94,6 +94,10 @@ public abstract class SchedulerNode {
protected Resource resourceAllocatedPendingLaunch =
Resource.newInstance(0, 0);
+ // The max amount of resources that can be allocated to opportunistic
+ // containers on the node, specified as a ratio to its capacity
+ private final float maxOverAllocationRatio;
+
private volatile Set<String> labels = null;
private volatile Set<NodeAttribute> nodeAttributes = null;
@@ -102,7 +106,7 @@ public abstract class SchedulerNode {
private volatile long lastHeartbeatMonotonicTime;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
- Set<String> labels) {
+ Set<String> labels, float maxOverAllocationRatio) {
this.rmNode = node;
this.rmContext = node.getRMContext();
this.unallocatedResource = Resources.clone(node.getTotalCapability());
@@ -114,10 +118,24 @@ public abstract class SchedulerNode {
}
this.labels = ImmutableSet.copyOf(labels);
this.lastHeartbeatMonotonicTime = Time.monotonicNow();
+ this.maxOverAllocationRatio = maxOverAllocationRatio;
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
+ this(node, usePortForNodeName, labels,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverAllocationRatio) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ maxOverAllocationRatio);
}
public SchedulerNode(RMNode node, boolean usePortForNodeName) {
- this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
}
public RMNode getRMNode() {
@@ -671,9 +689,11 @@ public abstract class SchedulerNode {
/**
* Get the amount of resources that can be allocated to opportunistic
- * containers in the case of overallocation. It is calculated as
+ * containers in the case of overallocation, calculated as
* node capacity - (node utilization + resources of allocated-yet-not-started
- * containers).
+ * containers), subject to the maximum amount of resources that can be
+ * allocated to opportunistic containers on the node specified as a ratio to
+ * its capacity.
* @return the amount of resources that are available to be allocated to
* opportunistic containers
*/
@@ -706,11 +726,21 @@ public abstract class SchedulerNode {
Resource resourceAllowedForOpportunisticContainers =
Resources.createResource(allowedMemory, allowedCpu);
- // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
- // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ // cap the total amount of resources allocated to OPPORTUNISTIC containers
+ Resource maxOverallocation = getMaxOverallocationAllowed();
+ Resources.subtractFrom(maxOverallocation, allocatedResourceOpportunistic);
+ resourceAllowedForOpportunisticContainers = Resources.componentwiseMin(
+ maxOverallocation, resourceAllowedForOpportunisticContainers);
+
return resourceAllowedForOpportunisticContainers;
}
+ private Resource getMaxOverallocationAllowed() {
+ long maxMemory = (long) (capacity.getMemorySize() * maxOverAllocationRatio);
+ int maxVcore = (int) (capacity.getVirtualCores() * maxOverAllocationRatio);
+ return Resource.newInstance(maxMemory, maxVcore);
+ }
+
private static class ContainerInfo {
private final RMContainer container;
private boolean launchedOnNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 95490f5..a53dda4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -66,10 +66,15 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ @VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
}
+ public FSSchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverallocationRatio) {
+ super(node, usePortForNodeName, maxOverallocationRatio);
+ }
/**
* Total amount of reserved resources including reservations and preempted
* containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 5d48fac..25782a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -774,7 +774,7 @@ public class FairScheduler extends
writeLock.lock();
try {
FSSchedulerNode schedulerNode = new FSSchedulerNode(node,
- usePortForNodeName);
+ usePortForNodeName, maxOverAllocationRatioPerNode);
nodeTracker.addNode(schedulerNode);
triggerUpdate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d028d77/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index fbb7243..9302f88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3143,6 +3143,113 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test that max overallocation per node is enforced by Fair Scheduler.
+ * @throws Exception
+ */
+ @Test
+ public void testMaxOverallocationPerNode() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ float maxOverallocationRatio = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO, 1.5f);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 1G of memory and 1 vcores and an overallocation
+ // threshold of 1.0f and 1.0f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(1f, 1f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 1), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is zero after the container runs
+ ContainerStatus containerStatus1 = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus1),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create a scheduling request that should get allocated an OPPORTUNISTIC
+ // container because the node utilization is zero
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // node utilization is still zero after the container runs
+ ContainerStatus containerStatus2 = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus2),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create another scheduling request that should not get any allocation
+ // because of the max overallocation on the node will be exceeded.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ maxOverallocationRatio);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org