You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ss...@apache.org on 2012/10/16 02:04:13 UTC
svn commit: r1398581 [7/9] - in
/hadoop/common/branches/MR-3902/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoo...
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Tue Oct 16 00:02:55 2012
@@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
import org.junit.Before;
@@ -154,9 +155,12 @@ public class TestRMAppAttemptTransitions
mock(ContainerAllocationExpirer.class);
amLivelinessMonitor = mock(AMLivelinessMonitor.class);
amFinishingMonitor = mock(AMLivelinessMonitor.class);
- rmContext = new RMContextImpl(new MemStore(), rmDispatcher,
- containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, new ApplicationTokenSecretManager(new Configuration()));
+ Configuration conf = new Configuration();
+ rmContext =
+ new RMContextImpl(new MemStore(), rmDispatcher,
+ containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
+ null, new ApplicationTokenSecretManager(conf),
+ new RMContainerTokenSecretManager(conf));
scheduler = mock(YarnScheduler.class);
masterService = mock(ApplicationMasterService.class);
@@ -174,7 +178,7 @@ public class TestRMAppAttemptTransitions
rmDispatcher.register(AMLauncherEventType.class,
new TestAMLauncherEventDispatcher());
- rmDispatcher.init(new Configuration());
+ rmDispatcher.init(conf);
rmDispatcher.start();
@@ -258,6 +262,10 @@ public class TestRMAppAttemptTransitions
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
+ // Check events
+ verify(masterService).
+ unregisterAttempt(applicationAttempt.getAppAttemptId());
+
// this works for unmanaged and managed AM's because this is actually doing
// verify(application).handle(anyObject());
verify(application).handle(any(RMAppRejectedEvent.class));
@@ -523,7 +531,8 @@ public class TestRMAppAttemptTransitions
// launch AM and verify attempt failed
applicationAttempt.handle(new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl"));
- testAppAttemptSubmittedToFailedState("Unmanaged AM must register after AM attempt reaches LAUNCHED state.");
+ testAppAttemptSubmittedToFailedState(
+ "Unmanaged AM must register after AM attempt reaches LAUNCHED state.");
}
@Test
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Tue Oct 16 00:02:55 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -247,12 +249,13 @@ public class TestCapacityScheduler {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
cs.setConf(new YarnConfiguration());
- cs.reinitialize(conf, null, null);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf)));
checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
conf.setCapacity(A, 80f);
conf.setCapacity(B, 20f);
- cs.reinitialize(conf, null,null);
+ cs.reinitialize(conf,null);
checkQueueCapacities(cs, 80f, 20f);
}
@@ -343,7 +346,8 @@ public class TestCapacityScheduler {
conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
- cs.reinitialize(conf, null, null);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf)));
}
@Test
@@ -353,7 +357,8 @@ public class TestCapacityScheduler {
setupQueueConfiguration(csConf);
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
- cs.reinitialize(csConf, null, null);
+ cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(csConf)));
RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Oct 16 00:02:55 2012
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTru
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -120,7 +121,7 @@ public class TestLeafQueue {
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
- cs.reinitialize(csConf, null, rmContext);
+ cs.reinitialize(csConf, rmContext);
}
private static final String A = "a";
@@ -1288,19 +1289,29 @@ public class TestLeafQueue {
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
priority, recordFactory));
app_0_requests_0.add(
- TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // one extra
+ TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, // one extra
priority, recordFactory));
app_0.updateResourceRequests(app_0_requests_0);
- assertEquals(1, app_0.getTotalRequiredResources(priority));
+ assertEquals(2, app_0.getTotalRequiredResources(priority));
String host_3 = "host_3"; // on rack_1
FiCaSchedulerNode node_3 = TestUtils.getMockNode(host_3, rack_1, 0, 8*GB);
+ // Rack-delay
+ doReturn(1).when(a).getNodeLocalityDelay();
+
+ // Shouldn't assign RACK_LOCAL yet
+ assignment = a.assignContainers(clusterResource, node_3);
+ assertEquals(1, app_0.getSchedulingOpportunities(priority));
+ assertEquals(2, app_0.getTotalRequiredResources(priority));
+ assertEquals(NodeType.NODE_LOCAL, assignment.getType()); // None->NODE_LOCAL
+
+ // Should assign RACK_LOCAL now
assignment = a.assignContainers(clusterResource, node_3);
verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_3),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
- assertEquals(0, app_0.getTotalRequiredResources(priority));
+ assertEquals(1, app_0.getTotalRequiredResources(priority));
assertEquals(NodeType.RACK_LOCAL, assignment.getType());
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Tue Oct 16 00:02:55 2012
@@ -23,8 +23,8 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.junit.Test;
public class TestQueueParsing {
@@ -42,7 +42,8 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(conf);
- capacityScheduler.reinitialize(conf, null, null);
+ capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null,
+ null, null, null, null, new RMContainerTokenSecretManager(conf)));
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
@@ -138,7 +139,7 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
- capacityScheduler.reinitialize(conf, null, null);
+ capacityScheduler.reinitialize(conf, null);
}
public void testMaxCapacity() throws Exception {
@@ -161,7 +162,7 @@ public class TestQueueParsing {
try {
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
- capacityScheduler.reinitialize(conf, null, null);
+ capacityScheduler.reinitialize(conf, null);
} catch (IllegalArgumentException iae) {
fail = true;
}
@@ -173,7 +174,7 @@ public class TestQueueParsing {
// Now this should work
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
- capacityScheduler.reinitialize(conf, null, null);
+ capacityScheduler.reinitialize(conf, null);
fail = false;
try {
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Tue Oct 16 00:02:55 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
public class TestUtils {
private static final Log LOG = LogFactory.getLog(TestUtils.class);
@@ -79,9 +80,11 @@ public class TestUtils {
ContainerAllocationExpirer cae =
new ContainerAllocationExpirer(nullDispatcher);
+ Configuration conf = new Configuration();
RMContext rmContext =
new RMContextImpl(null, nullDispatcher, cae, null, null, null,
- new ApplicationTokenSecretManager(new Configuration()));
+ new ApplicationTokenSecretManager(conf),
+ new RMContainerTokenSecretManager(conf));
return rmContext;
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Oct 16 00:02:55 2012
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -93,14 +94,14 @@ public class TestFairScheduler {
@Before
public void setUp() throws IOException {
scheduler = new FairScheduler();
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
Store store = StoreFactory.getStore(conf);
resourceManager = new ResourceManager(store);
resourceManager.init(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
}
@After
@@ -109,6 +110,13 @@ public class TestFairScheduler {
resourceManager = null;
}
+ private Configuration createConfiguration() {
+ Configuration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ return conf;
+ }
+
private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
ApplicationId appIdImpl = recordFactory.newRecordInstance(ApplicationId.class);
@@ -217,7 +225,9 @@ public class TestFairScheduler {
new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
scheduler.handle(updateEvent);
- assertEquals(512, scheduler.getQueueManager().getQueue("queue1").
+ // Asked for less than min_allocation.
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ scheduler.getQueueManager().getQueue("queue1").
getQueueSchedulable().getResourceUsage().getMemory());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
@@ -278,9 +288,9 @@ public class TestFairScheduler {
@Test
public void testUserAsDefaultQueue() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
createAppAttemptId(1, 1), "default", "user1");
scheduler.handle(appAddedEvent);
@@ -288,7 +298,7 @@ public class TestFairScheduler {
assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size());
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
createAppAttemptId(2, 1), "default", "user2");
scheduler.handle(appAddedEvent2);
@@ -299,9 +309,9 @@ public class TestFairScheduler {
@Test
public void testFairShareWithMinAlloc() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -354,31 +364,35 @@ public class TestFairScheduler {
ApplicationAttemptId id22 = createAppAttemptId(2, 2);
scheduler.addApplication(id22, "queue2", "user1");
- // First ask, queue1 requests 1024
+ int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB;
+
+ // First ask, queue1 requests 1 large (minReqSize * 2).
List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
- ResourceRequest request1 = createResourceRequest(1024, "*", 1, 1);
+ ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1);
ask1.add(request1);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>());
- // Second ask, queue2 requests 1024 + (2 * 512)
+ // Second ask, queue2 requests 1 large + (2 * minReqSize)
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
- ResourceRequest request2 = createResourceRequest(1024, "foo", 1, 1);
- ResourceRequest request3 = createResourceRequest(512, "bar", 1, 2);
+ ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1);
+ ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2);
ask2.add(request2);
ask2.add(request3);
scheduler.allocate(id21, ask2, new ArrayList<ContainerId>());
- // Third ask, queue2 requests 1024
+ // Third ask, queue2 requests 1 large
List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
- ResourceRequest request4 = createResourceRequest(1024, "*", 1, 1);
+ ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1);
ask3.add(request4);
scheduler.allocate(id22, ask3, new ArrayList<ContainerId>());
scheduler.update();
- assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").getQueueSchedulable().getDemand().getMemory());
- assertEquals(1024 + 1024 + (2 * 512), scheduler.getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand().getMemory());
-
+ assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1")
+ .getQueueSchedulable().getDemand().getMemory());
+ assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler
+ .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand()
+ .getMemory());
}
@Test
@@ -405,9 +419,9 @@ public class TestFairScheduler {
@Test
public void testAllocationFileParsing() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -508,9 +522,9 @@ public class TestFairScheduler {
@Test
public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -611,9 +625,9 @@ public class TestFairScheduler {
@Test
public void testIsStarvedForMinShare() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -670,9 +684,9 @@ public class TestFairScheduler {
@Test
public void testIsStarvedForFairShare() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -734,9 +748,9 @@ public class TestFairScheduler {
* now this means decreasing order of priority.
*/
public void testChoiceOfPreemptedContainers() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
@@ -867,11 +881,11 @@ public class TestFairScheduler {
* Tests the timing of decision to preempt tasks.
*/
public void testPreemptionDecision() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
MockClock clock = new MockClock();
scheduler.setClock(clock);
- scheduler.reinitialize(conf, null, resourceManager.getRMContext());
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Tue Oct 16 00:02:55 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -59,7 +61,10 @@ public class TestFifoScheduler {
public void setUp() throws Exception {
Store store = StoreFactory.getStore(new Configuration());
resourceManager = new ResourceManager(store);
- resourceManager.init(new Configuration());
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler.class, ResourceScheduler.class);
+ resourceManager.init(conf);
}
@After
@@ -87,10 +92,10 @@ public class TestFifoScheduler {
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
RMContext rmContext = new RMContextImpl(null, dispatcher, null,
- null, null, null, null);
+ null, null, null, null, null);
FifoScheduler schedular = new FifoScheduler();
- schedular.reinitialize(new Configuration(), null, rmContext);
+ schedular.reinitialize(new Configuration(), rmContext);
ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java Tue Oct 16 00:02:55 2012
@@ -66,6 +66,12 @@ public class TestDelegationTokenRenewer
private static Token<?> lastRenewed = null;
private static Token<?> tokenToRenewIn2Sec = null;
+ private static void reset() {
+ counter = 0;
+ lastRenewed = null;
+ tokenToRenewIn2Sec = null;
+ }
+
@Override
public boolean handleKind(Text kind) {
return KIND.equals(kind);
@@ -124,6 +130,7 @@ public class TestDelegationTokenRenewer
@Before
public void setUp() throws Exception {
+ Renewer.reset();
delegationTokenRenewer = new DelegationTokenRenewer();
delegationTokenRenewer.init(conf);
delegationTokenRenewer.start();
@@ -367,7 +374,7 @@ public class TestDelegationTokenRenewer
Credentials ts = new Credentials();
MyToken token1 = dfs.getDelegationToken(new Text("user1"));
-
+
//to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token1;
LOG.info("token="+token1+" should be renewed for 2 secs");
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Tue Oct 16 00:02:55 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -159,7 +160,7 @@ public class TestRMWebApp {
deactivatedNodesMap.put(node.getHostName(), node);
}
return new RMContextImpl(new MemStore(), null, null, null, null,
- null, null) {
+ null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return applicationsMaps;
@@ -199,7 +200,8 @@ public class TestRMWebApp {
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
- cs.reinitialize(conf, null, null);
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf)));
return cs;
}
@@ -275,7 +277,7 @@ public class TestRMWebApp {
FifoScheduler fs = new FifoScheduler();
fs.setConf(new YarnConfiguration());
- fs.reinitialize(conf, null, null);
+ fs.reinitialize(conf, null);
return fs;
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Oct 16 00:02:55 2012
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -74,7 +75,10 @@ public class TestRMWebServices extends J
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
- rm = new MockRM(new Configuration());
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
+ rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
bind(ApplicationACLsManager.class).toInstance(
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Tue Oct 16 00:02:55 2012
@@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -82,6 +84,8 @@ public class TestRMWebServicesApps exten
bind(GenericExceptionHandler.class);
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 2);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
@@ -276,9 +280,9 @@ public class TestRMWebServicesApps exten
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test",
+ "org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
@@ -355,9 +359,9 @@ public class TestRMWebServicesApps exten
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test",
+ "org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Tue Oct 16 00:02:55 2012
@@ -54,6 +54,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
+
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
@@ -228,9 +229,9 @@ public class TestRMWebServicesNodes exte
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.api.records.NodeState.BOGUSSTATE",
+ "org.apache.hadoop.yarn.api.records.NodeState.BOGUSSTATE",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Tue Oct 16 00:02:55 2012
@@ -44,6 +44,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Oct 16 00:02:55 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.ServerSocket;
import java.net.UnknownHostException;
import org.apache.commons.logging.Log;
@@ -51,7 +50,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.CompositeService;
@@ -104,6 +102,12 @@ public class MiniYARNCluster extends Com
nodeManagers[index] = new CustomNodeManager();
}
}
+
+ @Override
+ public void init(Configuration conf) {
+ super.init(conf instanceof YarnConfiguration ? conf
+ : new YarnConfiguration(conf));
+ }
public File getTestWorkDir() {
return testWorkDir;
@@ -203,7 +207,7 @@ public class MiniYARNCluster extends Com
}
public synchronized void init(Configuration conf) {
- Configuration config = new Configuration(conf);
+ Configuration config = new YarnConfiguration(conf);
super.init(config);
}
@@ -290,10 +294,9 @@ public class MiniYARNCluster extends Com
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher,
- healthChecker, metrics, containerTokenSecretManager) {
+ healthChecker, metrics) {
@Override
protected ResourceTracker getRMClient() {
final ResourceTrackerService rt = resourceManager
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Tue Oct 16 00:02:55 2012
@@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -218,9 +219,10 @@ public class TestContainerManagerSecurit
// Malice user modifies the resource amount
Resource modifiedResource = BuilderUtils.newResource(2048);
- ContainerTokenIdentifier modifiedIdentifier = new ContainerTokenIdentifier(
- dummyIdentifier.getContainerID(), dummyIdentifier.getNmHostAddress(),
- modifiedResource, Long.MAX_VALUE);
+ ContainerTokenIdentifier modifiedIdentifier =
+ new ContainerTokenIdentifier(dummyIdentifier.getContainerID(),
+ dummyIdentifier.getNmHostAddress(), "testUser", modifiedResource,
+ Long.MAX_VALUE, dummyIdentifier.getMasterKeyId());
Token<ContainerTokenIdentifier> modifiedToken = new Token<ContainerTokenIdentifier>(
modifiedIdentifier.getBytes(), containerToken.getPassword().array(),
new Text(containerToken.getKind()), new Text(containerToken
@@ -248,14 +250,14 @@ public class TestContainerManagerSecurit
+ "it will indicate RPC success");
} catch (Exception e) {
Assert.assertEquals(
- java.lang.reflect.UndeclaredThrowableException.class
- .getCanonicalName(), e.getClass().getCanonicalName());
+ java.lang.reflect.UndeclaredThrowableException.class
+ .getCanonicalName(), e.getClass().getCanonicalName());
Assert.assertTrue(e
- .getCause()
- .getMessage()
- .contains(
- "DIGEST-MD5: digest response format violation. "
- + "Mismatched response."));
+ .getCause()
+ .getMessage()
+ .contains(
+ "DIGEST-MD5: digest response format violation. "
+ + "Mismatched response."));
}
return null;
}
@@ -319,32 +321,35 @@ public class TestContainerManagerSecurit
callWithIllegalContainerID(client, tokenId);
callWithIllegalResource(client, tokenId);
+ callWithIllegalUserName(client, tokenId);
return client;
}
});
- /////////// End of testing for illegal containerIDs and illegal Resources
+ // ///////// End of testing for illegal containerIDs, illegal Resources and
+ // illegal users
/////////// Test calls with expired tokens
RPC.stopProxy(client);
unauthorizedUser = UserGroupInformation
.createRemoteUser(containerID.toString());
+ RMContainerTokenSecretManager containerTokenSecreteManager =
+ resourceManager.getRMContainerTokenSecretManager();
final ContainerTokenIdentifier newTokenId =
new ContainerTokenIdentifier(tokenId.getContainerID(),
- tokenId.getNmHostAddress(), tokenId.getResource(),
- System.currentTimeMillis() - 1);
+ tokenId.getNmHostAddress(), "testUser", tokenId.getResource(),
+ System.currentTimeMillis() - 1,
+ containerTokenSecreteManager.getCurrentKey().getKeyId());
byte[] passowrd =
- resourceManager.getContainerTokenSecretManager().createPassword(
+ containerTokenSecreteManager.createPassword(
newTokenId);
// Create a valid token by using the key from the RM.
token = new Token<ContainerTokenIdentifier>(
newTokenId.getBytes(), passowrd, new Text(
containerToken.getKind()), new Text(containerToken.getService()));
-
-
-
+
unauthorizedUser.addToken(token);
unauthorizedUser.doAs(new PrivilegedAction<Void>() {
@Override
@@ -563,6 +568,29 @@ public class TestContainerManagerSecurit
}
}
+ void callWithIllegalUserName(ContainerManager client,
+ ContainerTokenIdentifier tokenId) {
+ StartContainerRequest request = recordFactory
+ .newRecordInstance(StartContainerRequest.class);
+ // Authenticated but unauthorized, due to wrong resource
+ ContainerLaunchContext context =
+ createContainerLaunchContextForTest(tokenId);
+ context.setUser("Saruman"); // Set a different user-name.
+ request.setContainerLaunchContext(context);
+ try {
+ client.startContainer(request);
+ fail("Connection initiation with unauthorized "
+ + "access is expected to fail.");
+ } catch (YarnRemoteException e) {
+ LOG.info("Got exception : ", e);
+ Assert.assertTrue(e.getMessage().contains(
+ "Unauthorized request to start container. "));
+ Assert.assertTrue(e.getMessage().contains(
+ "Expected user-name " + tokenId.getApplicationSubmitter()
+ + " but found " + context.getUser()));
+ }
+ }
+
private ContainerLaunchContext createContainerLaunchContextForTest(
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Tue Oct 16 00:02:55 2012
@@ -278,17 +278,17 @@ Hadoop MapReduce Next Generation - Capac
| | Existing applications continue to completion, thus the queue can be
| | <drained> gracefully. |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.root.<queue-path>.acl_submit_jobs>>> | |
-| | The <ACL> which controls who can <submit> jobs to the given queue. |
+| <<<yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications>>> | |
+| | The <ACL> which controls who can <submit> applications to the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can submit jobs. |
+| | <one of the parent queues in the hierarchy> they can submit applications. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_jobs>>> | |
-| | The <ACL> which controls who can <administer> jobs on the given queue. |
+| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue>>> | |
+| | The <ACL> which controls who can <administer> applications on the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can administer jobs. |
+| | <one of the parent queues in the hierarchy> they can administer applications. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
@@ -318,7 +318,7 @@ Hadoop MapReduce Next Generation - Capac
----
$ vi $HADOOP_CONF_DIR/capacity-scheduler.xml
-$ $YARN_HOME/bin/yarn rmadmin -refreshQueues
+$ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues
----
<Note:> Queues cannot be <deleted>, only addition of new queues is supported -