You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [22/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Aug 19 23:49:39 2014
@@ -41,13 +41,14 @@ import java.util.List;
import java.util.Map;
import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -114,7 +115,7 @@ public class TestLeafQueue {
setupQueueConfiguration(csConf, newRoot);
YarnConfiguration conf = new YarnConfiguration();
cs.setConf(conf);
-
+
csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
when(csContext.getConf()).thenReturn(conf);
@@ -122,7 +123,7 @@ public class TestLeafQueue {
thenReturn(Resources.createResource(GB, 1));
when(csContext.getMaximumResourceCapability()).
thenReturn(Resources.createResource(16*GB, 32));
- when(csContext.getClusterResources()).
+ when(csContext.getClusterResource()).
thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
when(csContext.getApplicationComparator()).
thenReturn(CapacityScheduler.applicationComparator);
@@ -142,7 +143,9 @@ public class TestLeafQueue {
queues, queues,
TestUtils.spyHook);
- cs.reinitialize(csConf, rmContext);
+ cs.setRMContext(rmContext);
+ cs.init(csConf);
+ cs.start();
}
private static final String A = "a";
@@ -350,8 +353,7 @@ public class TestLeafQueue {
// Submit applications
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 1);
- FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null,
- rmContext);
+
AppAddedSchedulerEvent addAppEvent =
new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
a.getQueueName(), user_0);
@@ -497,8 +499,11 @@ public class TestLeafQueue {
// Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
- a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- null, RMContainerEventType.KILL, null);
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
}
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -508,8 +513,11 @@ public class TestLeafQueue {
// Release each container from app_1
for (RMContainer rmContainer : app_1.getLiveContainers()) {
- a.completedContainer(clusterResource, app_1, node_0, rmContainer,
- null, RMContainerEventType.KILL, null);
+ a.completedContainer(clusterResource, app_1, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
}
assertEquals(0*GB, a.getUsedResources().getMemory());
@@ -687,7 +695,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
- assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
+ assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G
// Again one to user_0 since he hasn't exceeded user limit yet
a.assignContainers(clusterResource, node_0);
@@ -695,7 +703,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
- assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
+ assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
@@ -869,8 +877,11 @@ public class TestLeafQueue {
// 8. Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
- a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- null, RMContainerEventType.KILL, null);
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
}
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -880,8 +891,11 @@ public class TestLeafQueue {
// 9. Release each container from app_2
for (RMContainer rmContainer : app_2.getLiveContainers()) {
- a.completedContainer(clusterResource, app_2, node_0, rmContainer,
- null, RMContainerEventType.KILL, null);
+ a.completedContainer(clusterResource, app_2, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
}
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -891,8 +905,11 @@ public class TestLeafQueue {
// 10. Release each container from app_3
for (RMContainer rmContainer : app_3.getLiveContainers()) {
- a.completedContainer(clusterResource, app_3, node_0, rmContainer,
- null, RMContainerEventType.KILL, null);
+ a.completedContainer(clusterResource, app_3, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
}
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -978,9 +995,12 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Now free 1 container from app_0 i.e. 1G
- a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(),
- null, RMContainerEventType.KILL, null);
+ RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
@@ -991,9 +1011,12 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
// Now finish another container from app_0 and fulfill the reservation
- a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(),
- null, RMContainerEventType.KILL, null);
+ rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1093,9 +1116,12 @@ public class TestLeafQueue {
assertEquals(6*GB, a.getMetrics().getAllocatedMB());
// Now free 1 container from app_0 and try to assign to node_0
- a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(),
- null, RMContainerEventType.KILL, null);
+ RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(8*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1185,9 +1211,12 @@ public class TestLeafQueue {
assertEquals(2*GB, node_0.getUsedResource().getMemory());
// Now free 1 container from app_0 i.e. 1G, and re-reserve it
- a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(),
- null, RMContainerEventType.KILL, null);
+ RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
@@ -1217,9 +1246,12 @@ public class TestLeafQueue {
assertEquals(2, app_1.getReReservations(priority));
// Now finish another container from app_0 and see the reservation cancelled
- a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(),
- null, RMContainerEventType.KILL, null);
+ rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null);
CSAssignment assignment = a.assignContainers(clusterResource, node_0);
assertEquals(8*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1652,7 +1684,7 @@ public class TestLeafQueue {
newQueues, queues,
TestUtils.spyHook);
queues = newQueues;
- root.reinitialize(newRoot, cs.getClusterResources());
+ root.reinitialize(newRoot, cs.getClusterResource());
// after reinitialization
assertEquals(3, e.activeApplications.size());
@@ -1677,7 +1709,7 @@ public class TestLeafQueue {
newQueues, queues,
TestUtils.spyHook);
queues = newQueues;
- root.reinitialize(newRoot, cs.getClusterResources());
+ root.reinitialize(newRoot, cs.getClusterResource());
// after reinitialization
assertEquals(60, e.getNodeLocalityDelay());
@@ -2071,7 +2103,7 @@ public class TestLeafQueue {
when(csContext.getConfiguration()).thenReturn(csConf);
when(csContext.getConf()).thenReturn(new YarnConfiguration());
when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
- when(csContext.getClusterResources()).thenReturn(clusterResource);
+ when(csContext.getClusterResource()).thenReturn(clusterResource);
when(csContext.getMinimumResourceCapability()).thenReturn(
Resources.createResource(GB, 1));
when(csContext.getMaximumResourceCapability()).thenReturn(
@@ -2081,5 +2113,8 @@ public class TestLeafQueue {
@After
public void tearDown() throws Exception {
+ if (cs != null) {
+ cs.stop();
+ }
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Tue Aug 19 23:49:39 2014
@@ -86,7 +86,7 @@ public class TestParentQueue {
Resources.createResource(GB, 1));
when(csContext.getMaximumResourceCapability()).thenReturn(
Resources.createResource(16*GB, 32));
- when(csContext.getClusterResources()).
+ when(csContext.getClusterResource()).
thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
when(csContext.getApplicationComparator()).
thenReturn(CapacityScheduler.applicationComparator);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Tue Aug 19 23:49:39 2014
@@ -43,11 +43,15 @@ public class TestQueueParsing {
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext = new RMContextImpl(null, null,
+ null, null, null, null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null);
capacityScheduler.setConf(conf);
- capacityScheduler.reinitialize(conf, new RMContextImpl(null, null,
- null, null, null, null, new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), null));
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
+ capacityScheduler.reinitialize(conf, rmContext);
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
@@ -62,6 +66,7 @@ public class TestQueueParsing {
Assert.assertEquals(0.7 * 0.5 * 0.45, c12.getAbsoluteCapacity(), DELTA);
Assert.assertEquals(0.7 * 0.55 * 0.7,
c12.getAbsoluteMaximumCapacity(), DELTA);
+ capacityScheduler.stop();
}
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
@@ -142,7 +147,10 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
+ capacityScheduler.stop();
}
public void testMaxCapacity() throws Exception {
@@ -164,6 +172,8 @@ public class TestQueueParsing {
try {
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
} catch (IllegalArgumentException iae) {
fail = true;
@@ -176,6 +186,8 @@ public class TestQueueParsing {
// Now this should work
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
fail = false;
@@ -187,6 +199,7 @@ public class TestQueueParsing {
}
Assert.assertTrue("Didn't throw IllegalArgumentException for wrong " +
"setMaxCap", fail);
+ capacityScheduler.stop();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Tue Aug 19 23:49:39 2014
@@ -86,13 +86,12 @@ public class TestUtils {
Configuration conf = new Configuration();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
- RMContext rmContext =
+ RMContextImpl rmContext =
new RMContextImpl(nullDispatcher, cae, null, null, null,
- new AMRMTokenSecretManager(conf),
+ new AMRMTokenSecretManager(conf, null),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), writer);
-
return rmContext;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Tue Aug 19 23:49:39 2014
@@ -21,16 +21,18 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
* Dummy implementation of Schedulable for unit testing.
*/
-public class FakeSchedulable extends Schedulable {
+public class FakeSchedulable implements Schedulable {
private Resource usage;
private Resource minShare;
private Resource maxShare;
+ private Resource fairShare;
private ResourceWeights weights;
private Priority priority;
private long startTime;
@@ -84,6 +86,26 @@ public class FakeSchedulable extends Sch
}
@Override
+ public RMContainer preemptContainer() {
+ return null;
+ }
+
+ @Override
+ public Resource getFairShare() {
+ return this.fairShare;
+ }
+
+ @Override
+ public void setFairShare(Resource fairShare) {
+ this.fairShare = fairShare;
+ }
+
+ @Override
+ public boolean isActive() {
+ return true;
+ }
+
+ @Override
public Resource getDemand() {
return null;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java Tue Aug 19 23:49:39 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.FileWriter;
@@ -28,6 +27,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -99,9 +99,12 @@ public class TestAllocationFileLoaderSer
assertEquals(1, rules.size());
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
- assertEquals(2, allocConf.getQueueNames().size());
- assertTrue(allocConf.getQueueNames().contains("root.queueA"));
- assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+ assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .size());
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueA"));
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueB"));
confHolder.allocConf = null;
@@ -114,6 +117,9 @@ public class TestAllocationFileLoaderSer
out.println(" </queue>");
out.println(" <queuePlacementPolicy>");
out.println(" <rule name='specified' />");
+ out.println(" <rule name='nestedUserQueue' >");
+ out.println(" <rule name='primaryGroup' />");
+ out.println(" </rule>");
out.println(" <rule name='default' />");
out.println(" </queuePlacementPolicy>");
out.println("</allocations>");
@@ -131,12 +137,18 @@ public class TestAllocationFileLoaderSer
allocConf = confHolder.allocConf;
policy = allocConf.getPlacementPolicy();
rules = policy.getRules();
- assertEquals(2, rules.size());
+ assertEquals(3, rules.size());
assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass());
- assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass());
+ assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1)
+ .getClass());
+ assertEquals(QueuePlacementRule.PrimaryGroup.class,
+ ((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
+ assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
- assertEquals(1, allocConf.getQueueNames().size());
- assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+ assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .size());
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueB"));
}
@Test
@@ -162,18 +174,29 @@ public class TestAllocationFileLoaderSer
out.println("<queue name=\"queueC\">");
out.println("<aclSubmitApps>alice,bob admins</aclSubmitApps>");
out.println("</queue>");
- // Give queue D a limit of 3 running apps
+ // Give queue D a limit of 3 running apps and 0.4f maxAMShare
out.println("<queue name=\"queueD\">");
out.println("<maxRunningApps>3</maxRunningApps>");
+ out.println("<maxAMShare>0.4</maxAMShare>");
out.println("</queue>");
// Give queue E a preemption timeout of one minute
out.println("<queue name=\"queueE\">");
out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
out.println("</queue>");
+ //Make queue F a parent queue without configured leaf queues using the 'type' attribute
+ out.println("<queue name=\"queueF\" type=\"parent\" >");
+ out.println("</queue>");
+ //Create hierarchical queues G,H
+ out.println("<queue name=\"queueG\">");
+ out.println(" <queue name=\"queueH\">");
+ out.println(" </queue>");
+ out.println("</queue>");
// Set default limit of apps per queue to 15
out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
// Set default limit of apps per user to 5
out.println("<userMaxAppsDefault>5</userMaxAppsDefault>");
+ // Set default limit of AMResourceShare to 0.5f
+ out.println("<queueMaxAMShareDefault>0.5f</queueMaxAMShareDefault>");
// Give user1 a limit of 10 jobs
out.println("<user name=\"user1\">");
out.println("<maxRunningApps>10</maxRunningApps>");
@@ -194,7 +217,7 @@ public class TestAllocationFileLoaderSer
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
- assertEquals(5, queueConf.getQueueNames().size());
+ assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
@@ -220,6 +243,13 @@ public class TestAllocationFileLoaderSer
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root." + YarnConfiguration.DEFAULT_QUEUE_NAME), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueA"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueB"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueC"), 0.01);
+ assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01);
+
// Root should get * ACL
assertEquals("*", queueConf.getQueueAcl("root",
QueueACL.ADMINISTER_QUEUE).getAclString());
@@ -250,6 +280,14 @@ public class TestAllocationFileLoaderSer
assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueConf.getFairSharePreemptionTimeout());
+ assertTrue(queueConf.getConfiguredQueues()
+ .get(FSQueueType.PARENT)
+ .contains("root.queueF"));
+ assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT)
+ .contains("root.queueG"));
+ assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueG.queueH"));
+
// Verify existing queues have default scheduling policy
assertEquals(DominantResourceFairnessPolicy.NAME,
queueConf.getSchedulingPolicy("root").getName());
@@ -315,7 +353,7 @@ public class TestAllocationFileLoaderSer
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
- assertEquals(5, queueConf.getQueueNames().size());
+ assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java Tue Aug 19 23:49:39 2014
@@ -48,6 +48,8 @@ public class TestFSLeafQueue {
ResourceManager resourceManager = new ResourceManager();
resourceManager.init(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
String queueName = "root.queue1";
@@ -60,7 +62,7 @@ public class TestFSLeafQueue {
@Test
public void testUpdateDemand() {
- AppSchedulable app = mock(AppSchedulable.class);
+ FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(maxResource);
schedulable.addAppSchedulable(app);