You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ac...@apache.org on 2013/01/09 06:17:34 UTC
svn commit: r1430683 [3/3] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/...
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Wed Jan 9 05:17:31 2013
@@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -68,6 +70,9 @@ public class TestParentQueue {
final static int GB = 1024;
final static String DEFAULT_RACK = "/default";
+ private final ResourceCalculator resourceComparator =
+ new DefaultResourceCalculator();
+
@Before
public void setUp() throws Exception {
rmContext = TestUtils.getMockRMContext();
@@ -78,11 +83,17 @@ public class TestParentQueue {
when(csContext.getConf()).thenReturn(conf);
when(csContext.getConfiguration()).thenReturn(csConf);
when(csContext.getMinimumResourceCapability()).thenReturn(
- Resources.createResource(GB));
+ Resources.createResource(GB, 1));
when(csContext.getMaximumResourceCapability()).thenReturn(
- Resources.createResource(16*GB));
+ Resources.createResource(16*GB, 32));
when(csContext.getClusterResources()).
- thenReturn(Resources.createResource(100 * 16 * GB));
+ thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
+ when(csContext.getApplicationComparator()).
+ thenReturn(CapacityScheduler.applicationComparator);
+ when(csContext.getQueueComparator()).
+ thenReturn(CapacityScheduler.queueComparator);
+ when(csContext.getResourceCalculator()).
+ thenReturn(resourceComparator);
}
private static final String A = "a";
@@ -104,7 +115,7 @@ public class TestParentQueue {
private FiCaSchedulerApp getMockApplication(int appId, String user) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
doReturn(user).when(application).getUser();
- doReturn(Resources.createResource(0)).when(application).getHeadroom();
+ doReturn(Resources.createResource(0, 0)).when(application).getHeadroom();
return application;
}
@@ -192,12 +203,11 @@ public class TestParentQueue {
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
// Setup some nodes
final int memoryPerNode = 10;
+ final int coresPerNode = 16;
final int numNodes = 2;
FiCaSchedulerNode node_0 =
@@ -206,7 +216,8 @@ public class TestParentQueue {
TestUtils.getMockNode("host_1", DEFAULT_RACK, 0, memoryPerNode*GB);
final Resource clusterResource =
- Resources.createResource(numNodes * (memoryPerNode*GB));
+ Resources.createResource(numNodes * (memoryPerNode*GB),
+ numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
// Start testing
@@ -286,8 +297,7 @@ public class TestParentQueue {
try {
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator, TestUtils.spyHook);
+ TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
exceptionOccured = true;
}
@@ -301,8 +311,7 @@ public class TestParentQueue {
try {
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator, TestUtils.spyHook);
+ TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
exceptionOccured = true;
}
@@ -316,8 +325,7 @@ public class TestParentQueue {
try {
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator, TestUtils.spyHook);
+ TestUtils.spyHook);
} catch (IllegalArgumentException ie) {
exceptionOccured = true;
}
@@ -394,12 +402,11 @@ public class TestParentQueue {
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
// Setup some nodes
final int memoryPerNode = 10;
+ final int coresPerNode = 16;
final int numNodes = 3;
FiCaSchedulerNode node_0 =
@@ -410,7 +417,8 @@ public class TestParentQueue {
TestUtils.getMockNode("host_2", DEFAULT_RACK, 0, memoryPerNode*GB);
final Resource clusterResource =
- Resources.createResource(numNodes * (memoryPerNode*GB));
+ Resources.createResource(numNodes * (memoryPerNode*GB),
+ numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
// Start testing
@@ -510,8 +518,6 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
}
@@ -529,8 +535,6 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
}
@@ -553,8 +557,6 @@ public class TestParentQueue {
try {
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
} catch (IllegalArgumentException e) {
fail("Failed to create queues with 0 capacity: " + e);
@@ -571,12 +573,11 @@ public class TestParentQueue {
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
// Setup some nodes
final int memoryPerNode = 10;
+ final int coresPerNode = 16;
final int numNodes = 2;
FiCaSchedulerNode node_0 =
@@ -585,7 +586,8 @@ public class TestParentQueue {
TestUtils.getMockNode("host_1", DEFAULT_RACK, 0, memoryPerNode*GB);
final Resource clusterResource =
- Resources.createResource(numNodes * (memoryPerNode*GB));
+ Resources.createResource(numNodes * (memoryPerNode*GB),
+ numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
// Start testing
@@ -637,12 +639,11 @@ public class TestParentQueue {
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
// Setup some nodes
final int memoryPerNode = 10;
+ final int coresPerNode = 10;
final int numNodes = 2;
FiCaSchedulerNode node_0 =
@@ -651,7 +652,8 @@ public class TestParentQueue {
TestUtils.getMockNode("host_1", DEFAULT_RACK, 0, memoryPerNode*GB);
final Resource clusterResource =
- Resources.createResource(numNodes * (memoryPerNode*GB));
+ Resources.createResource(numNodes * (memoryPerNode*GB),
+ numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
// Start testing
@@ -721,8 +723,6 @@ public class TestParentQueue {
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
- CapacityScheduler.queueComparator,
- CapacityScheduler.applicationComparator,
TestUtils.spyHook);
UserGroupInformation user = UserGroupInformation.getCurrentUser();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Wed Jan 9 05:17:31 2013
@@ -118,7 +118,7 @@ public class TestUtils {
RecordFactory recordFactory) {
ResourceRequest request =
recordFactory.newRecordInstance(ResourceRequest.class);
- Resource capability = Resources.createResource(memory);
+ Resource capability = Resources.createResource(memory, 1);
request.setNumContainers(numContainers);
request.setHostName(hostName);
@@ -153,7 +153,7 @@ public class TestUtils {
RMNode rmNode = mock(RMNode.class);
when(rmNode.getNodeID()).thenReturn(nodeId);
when(rmNode.getTotalCapability()).thenReturn(
- Resources.createResource(capability));
+ Resources.createResource(capability, 1));
when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
when(rmNode.getHostName()).thenReturn(host);
when(rmNode.getRackName()).thenReturn(rack);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Wed Jan 9 05:17:31 2013
@@ -71,9 +71,9 @@ public class TestFifoScheduler {
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
registerNode(String hostName, int containerManagerPort, int nmHttpPort,
- String rackName, int memory) throws IOException {
+ String rackName, Resource capability) throws IOException {
return new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
- hostName, containerManagerPort, nmHttpPort, rackName, memory,
+ hostName, containerManagerPort, nmHttpPort, rackName, capability,
resourceManager.getResourceTrackerService(), resourceManager
.getRMContext());
}
@@ -121,13 +121,15 @@ public class TestFifoScheduler {
// Register node1
String host_0 = "host_0";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
- registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, 4 * GB);
+ registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+ Resources.createResource(4 * GB, 1));
nm_0.heartbeat();
// Register node2
String host_1 = "host_1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
- registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, 2 * GB);
+ registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+ Resources.createResource(2 * GB, 1));
nm_1.heartbeat();
// ResourceRequest priorities
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java Wed Jan 9 05:17:31 2013
@@ -424,8 +424,8 @@ public class TestRMWebServicesCapacitySc
assertEquals("absoluteUsedCapacity doesn't match",
0, info.absoluteUsedCapacity, 1e-3f);
assertEquals("numApplications doesn't match", 0, info.numApplications);
- assertTrue("usedResources doesn't match",
- info.usedResources.matches("memory: 0"));
+ assertTrue("usedResources doesn't match ",
+ info.usedResources.matches("<memory:0, vCores:0>"));
assertTrue("queueName doesn't match, got: " + info.queueName
+ " expected: " + q, qshortName.matches(info.queueName));
assertTrue("state doesn't match",
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed Jan 9 05:17:31 2013
@@ -218,7 +218,7 @@ public class TestContainerManagerSecurit
dummyIdentifier.readFields(di);
// Malice user modifies the resource amount
- Resource modifiedResource = BuilderUtils.newResource(2048);
+ Resource modifiedResource = BuilderUtils.newResource(2048, 1);
ContainerTokenIdentifier modifiedIdentifier =
new ContainerTokenIdentifier(dummyIdentifier.getContainerID(),
dummyIdentifier.getNmHostAddress(), "testUser", modifiedResource,
@@ -403,7 +403,7 @@ public class TestContainerManagerSecurit
ContainerLaunchContext amContainer = BuilderUtils
.newContainerLaunchContext(null, "testUser", BuilderUtils
- .newResource(1024), Collections.<String, LocalResource>emptyMap(),
+ .newResource(1024, 1), Collections.<String, LocalResource>emptyMap(),
new HashMap<String, String>(), Arrays.asList("sleep", "100"),
new HashMap<String, ByteBuffer>(), null,
new HashMap<ApplicationAccessType, String>());
@@ -481,7 +481,7 @@ public class TestContainerManagerSecurit
// Request a container allocation.
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*",
- BuilderUtils.newResource(1024), 1));
+ BuilderUtils.newResource(1024, 1), 1));
AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest(
BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask,
@@ -582,7 +582,9 @@ public class TestContainerManagerSecurit
ContainerLaunchContext context =
BuilderUtils.newContainerLaunchContext(tokenId.getContainerID(),
"testUser",
- BuilderUtils.newResource(tokenId.getResource().getMemory()),
+ BuilderUtils.newResource(
+ tokenId.getResource().getMemory(),
+ tokenId.getResource().getVirtualCores()),
new HashMap<String, LocalResource>(),
new HashMap<String, String>(), new ArrayList<String>(),
new HashMap<String, ByteBuffer>(), null,