You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/24 19:07:47 UTC
[25/50] [abbrv] hadoop git commit: YARN-4511. Common scheduler
changes to support scheduler-specific oversubscription implementations.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 1836919..02c0cc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -563,7 +563,7 @@ public class TestNodeLabelContainerAllocation {
int numContainers) {
CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
SchedulerNode node = cs.getSchedulerNode(nodeId);
- Assert.assertEquals(numContainers, node.getNumContainers());
+ Assert.assertEquals(numContainers, node.getNumGuaranteedContainers());
}
/**
@@ -1065,7 +1065,7 @@ public class TestNodeLabelContainerAllocation {
for (int i = 0; i < 50; i++) {
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
- if (schedulerNode1.getNumContainers() == 0) {
+ if (schedulerNode1.getNumGuaranteedContainers() == 0) {
cycleWaited++;
}
}
@@ -1131,7 +1131,7 @@ public class TestNodeLabelContainerAllocation {
CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG
+ nodeIdStr + " ( Partition : [x]"));
Assert.assertEquals(0, cs.getSchedulerNode(nm1.getNodeId())
- .getNumContainers());
+ .getNumGuaranteedContainers());
rm1.close();
}
@@ -1215,7 +1215,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(10, schedulerNode1.getNumContainers());
+ Assert.assertEquals(10, schedulerNode1.getNumGuaranteedContainers());
// check non-exclusive containers of LeafQueue is correctly updated
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
@@ -1943,7 +1943,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x
- Assert.assertEquals(5, schedulerNode1.getNumContainers());
+ Assert.assertEquals(5, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2043,7 +2043,7 @@ public class TestNodeLabelContainerAllocation {
}
// app1 gets all resource in partition=x (non-exclusive)
- Assert.assertEquals(3, schedulerNode1.getNumContainers());
+ Assert.assertEquals(3, schedulerNode1.getNumGuaranteedContainers());
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
@@ -2074,7 +2074,7 @@ public class TestNodeLabelContainerAllocation {
cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
// app1 gets all resource in default partition
- Assert.assertEquals(2, schedulerNode2.getNumContainers());
+ Assert.assertEquals(2, schedulerNode2.getNumGuaranteedContainers());
// 3GB is used from label x quota. 2GB used from default label.
// So total 2.5 GB is remaining.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 2512787..6390297 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -20,16 +20,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +58,9 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import org.junit.Before;
import org.junit.Test;
@@ -324,7 +336,9 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
for (FSSchedulerNode node : clusterNodeTracker.getAllNodes()) {
int i = ThreadLocalRandom.current().nextInt(-30, 30);
synchronized (scheduler) {
- node.deductUnallocatedResource(Resource.newInstance(i * 1024, i));
+ RMContainer container = createRMContainer(
+ node.getRMNode(), Resource.newInstance(i * 1024, 1));
+ node.allocateContainer(container);
}
}
}
@@ -338,6 +352,32 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
}
}
+ private static RMContainer createRMContainer(
+ RMNode node, Resource resource) {
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+ newInstance(ApplicationId.newInstance(0, 0), 0);
+ ContainerId cId =
+ ContainerId.newContainerId(appAttemptId, 0);
+ Container container = Container.newInstance(
+ cId, node.getNodeID(), node.getNodeAddress(), resource,
+ Priority.newInstance(0), null, ExecutionType.GUARANTEED);
+
+ Dispatcher dispatcher = new AsyncDispatcher();
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ when(rmContext.getSystemMetricsPublisher()).
+ thenReturn(new NoOpSystemMetricPublisher());
+ when(rmContext.getYarnConfiguration()).
+ thenReturn(new YarnConfiguration());
+ when(rmContext.getContainerAllocationExpirer()).
+ thenReturn(new ContainerAllocationExpirer(dispatcher));
+ when(rmContext.getRMApplicationHistoryWriter()).
+ thenReturn(new RMApplicationHistoryWriter());
+ return new RMContainerImpl(container, null,
+ container.getId().getApplicationAttemptId(),
+ node.getNodeID(), "test", rmContext);
+ }
+
@Test
public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 0e3d344..6726f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -87,7 +87,7 @@ public class TestFSSchedulerNode {
while (!Resources.isNone(schedulerNode.getUnallocatedResource())) {
createDefaultContainer();
schedulerNode.allocateContainer(containers.get(containers.size() - 1));
- schedulerNode.containerStarted(containers.get(containers.size() - 1).
+ schedulerNode.containerLaunched(containers.get(containers.size() - 1).
getContainerId());
}
}
@@ -183,9 +183,9 @@ public class TestFSSchedulerNode {
assertEquals("Nothing should have been allocated, yet",
Resources.none(), schedulerNode.getAllocatedResource());
schedulerNode.allocateContainer(containers.get(0));
- schedulerNode.containerStarted(containers.get(0).getContainerId());
+ schedulerNode.containerLaunched(containers.get(0).getContainerId());
schedulerNode.allocateContainer(containers.get(1));
- schedulerNode.containerStarted(containers.get(1).getContainerId());
+ schedulerNode.containerLaunched(containers.get(1).getContainerId());
schedulerNode.allocateContainer(containers.get(2));
assertEquals("Container should be allocated",
Resources.multiply(containers.get(0).getContainer().getResource(), 3.0),
@@ -225,7 +225,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -266,7 +266,7 @@ public class TestFSSchedulerNode {
schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all remaining containers
@@ -312,7 +312,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -360,7 +360,7 @@ public class TestFSSchedulerNode {
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -399,7 +399,7 @@ public class TestFSSchedulerNode {
when(starvingApp.isStopped()).thenReturn(true);
allocateContainers(schedulerNode);
assertNotEquals("Container should be allocated",
- schedulerNode.getTotalResource(),
+ schedulerNode.getCapacity(),
schedulerNode.getAllocatedResource());
// Release all containers
@@ -437,7 +437,7 @@ public class TestFSSchedulerNode {
// Container partially reassigned
allocateContainers(schedulerNode);
assertEquals("Container should be allocated",
- Resources.subtract(schedulerNode.getTotalResource(),
+ Resources.subtract(schedulerNode.getCapacity(),
Resource.newInstance(512, 0)),
schedulerNode.getAllocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9120d3a..4b637ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3299,12 +3299,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(node2UpdateEvent);
if (invalid) {
assertEquals(0, app.getLiveContainers().size());
- assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(0,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
} else {
assertEquals(1, app.getLiveContainers().size());
- assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
- assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+ assertEquals(1,
+ scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+ assertEquals(0,
+ scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
}
}
@@ -5051,7 +5055,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Check total resource of scheduler node is also changed to 0 GB 0 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 0);
// Check the available resource is 0/0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce4c4a70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index ee66a49..3f8a552 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -344,7 +344,7 @@ public class TestFifoScheduler {
// SchedulerNode's total resource and available resource are changed.
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID())
- .getTotalResource().getMemorySize());
+ .getCapacity().getMemorySize());
assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).
getUnallocatedResource().getMemorySize(), 1024);
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
@@ -1293,7 +1293,7 @@ public class TestFifoScheduler {
// Check total resource of scheduler node is also changed to 1 GB 1 core
Resource totalResource =
resourceManager.getResourceScheduler()
- .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+ .getSchedulerNode(nm_0.getNodeId()).getCapacity();
Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
Assert.assertEquals(totalResource.getVirtualCores(), 1);
// Check the available resource is 0/0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org