You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cl...@apache.org on 2014/06/16 20:14:12 UTC
svn commit: r1602947 [5/5] - in
/hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/ha...
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java Mon Jun 16 18:13:57 2014
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
@@ -213,9 +213,8 @@ public class TestFSRMStateStore extends
try {
store.storeApplicationStateInternal(
ApplicationId.newInstance(100L, 1),
- (ApplicationStateDataPBImpl) ApplicationStateDataPBImpl
- .newApplicationStateData(111, 111, "user", null,
- RMAppState.ACCEPTED, "diagnostics", 333));
+ ApplicationStateData.newInstance(111, 111, "user", null,
+ RMAppState.ACCEPTED, "diagnostics", 333));
} catch (Exception e) {
// TODO 0 datanode exception will not be retried by dfs client, fix
// that separately.
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Mon Jun 16 18:13:57 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -921,6 +922,7 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
+ Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Mon Jun 16 18:13:57 2014
@@ -121,13 +121,16 @@ public class TestCapacityScheduler {
@After
public void tearDown() throws Exception {
- resourceManager.stop();
+ if (resourceManager != null) {
+ resourceManager.stop();
+ }
}
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
ResourceScheduler scheduler = new CapacityScheduler();
+ scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
@@ -342,18 +345,23 @@ public class TestCapacityScheduler {
public void testRefreshQueues() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null);
setupQueueConfiguration(conf);
cs.setConf(new YarnConfiguration());
- cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
- null, new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), null));
+ cs.setRMContext(resourceManager.getRMContext());
+ cs.init(conf);
+ cs.start();
+ cs.reinitialize(conf, rmContext);
checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
conf.setCapacity(A, 80f);
conf.setCapacity(B, 20f);
cs.reinitialize(conf, mockContext);
checkQueueCapacities(cs, 80f, 20f);
+ cs.stop();
}
private void checkQueueCapacities(CapacityScheduler cs,
@@ -456,6 +464,9 @@ public class TestCapacityScheduler {
setupQueueConfiguration(csConf);
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
+ cs.setRMContext(resourceManager.getRMContext());
+ cs.init(csConf);
+ cs.start();
cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
null, null, new RMContainerTokenSecretManager(csConf),
new NMTokenSecretManagerInRM(csConf),
@@ -475,6 +486,7 @@ public class TestCapacityScheduler {
cs.handle(new NodeAddedSchedulerEvent(n1));
Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
+ cs.stop();
}
@Test
@@ -483,6 +495,9 @@ public class TestCapacityScheduler {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
cs.setConf(new YarnConfiguration());
+ cs.setRMContext(resourceManager.getRMContext());
+ cs.init(conf);
+ cs.start();
cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
@@ -513,6 +528,7 @@ public class TestCapacityScheduler {
assertEquals(queueB, queueB4.getParent());
} finally {
B3_CAPACITY += B4_CAPACITY;
+ cs.stop();
}
}
@Test
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon Jun 16 18:13:57 2014
@@ -114,7 +114,7 @@ public class TestLeafQueue {
setupQueueConfiguration(csConf, newRoot);
YarnConfiguration conf = new YarnConfiguration();
cs.setConf(conf);
-
+
csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
when(csContext.getConf()).thenReturn(conf);
@@ -142,7 +142,9 @@ public class TestLeafQueue {
queues, queues,
TestUtils.spyHook);
- cs.reinitialize(csConf, rmContext);
+ cs.setRMContext(rmContext);
+ cs.init(csConf);
+ cs.start();
}
private static final String A = "a";
@@ -2080,5 +2082,8 @@ public class TestLeafQueue {
@After
public void tearDown() throws Exception {
+ if (cs != null) {
+ cs.stop();
+ }
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Mon Jun 16 18:13:57 2014
@@ -43,11 +43,15 @@ public class TestQueueParsing {
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext = new RMContextImpl(null, null,
+ null, null, null, null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null);
capacityScheduler.setConf(conf);
- capacityScheduler.reinitialize(conf, new RMContextImpl(null, null,
- null, null, null, null, new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), null));
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
+ capacityScheduler.reinitialize(conf, rmContext);
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
@@ -62,6 +66,7 @@ public class TestQueueParsing {
Assert.assertEquals(0.7 * 0.5 * 0.45, c12.getAbsoluteCapacity(), DELTA);
Assert.assertEquals(0.7 * 0.55 * 0.7,
c12.getAbsoluteMaximumCapacity(), DELTA);
+ capacityScheduler.stop();
}
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
@@ -142,7 +147,10 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
+ capacityScheduler.stop();
}
public void testMaxCapacity() throws Exception {
@@ -164,6 +172,8 @@ public class TestQueueParsing {
try {
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
} catch (IllegalArgumentException iae) {
fail = true;
@@ -176,6 +186,8 @@ public class TestQueueParsing {
// Now this should work
capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(new YarnConfiguration());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
capacityScheduler.reinitialize(conf, null);
fail = false;
@@ -187,6 +199,7 @@ public class TestQueueParsing {
}
Assert.assertTrue("Didn't throw IllegalArgumentException for wrong " +
"setMaxCap", fail);
+ capacityScheduler.stop();
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java Mon Jun 16 18:13:57 2014
@@ -20,14 +20,21 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
@@ -169,4 +176,20 @@ public class FairSchedulerTestBase {
ask.add(request);
scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null);
}
+
+ protected void createApplicationWithAMResource(ApplicationAttemptId attId,
+ String queue, String user, Resource amResource) {
+ RMContext rmContext = resourceManager.getRMContext();
+ RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf,
+ null, null, null, ApplicationSubmissionContext.newInstance(null, null,
+ null, null, null, false, false, 0, amResource, null), null, null,
+ 0, null, null);
+ rmContext.getRMApps().put(attId.getApplicationId(), rmApp);
+ AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
+ attId.getApplicationId(), queue, user);
+ scheduler.handle(appAddedEvent);
+ AppAttemptAddedSchedulerEvent attempAddedEvent =
+ new AppAttemptAddedSchedulerEvent(attId, false);
+ scheduler.handle(attempAddedEvent);
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java Mon Jun 16 18:13:57 2014
@@ -174,9 +174,10 @@ public class TestAllocationFileLoaderSer
out.println("<queue name=\"queueC\">");
out.println("<aclSubmitApps>alice,bob admins</aclSubmitApps>");
out.println("</queue>");
- // Give queue D a limit of 3 running apps
+ // Give queue D a limit of 3 running apps and 0.4f maxAMShare
out.println("<queue name=\"queueD\">");
out.println("<maxRunningApps>3</maxRunningApps>");
+ out.println("<maxAMShare>0.4</maxAMShare>");
out.println("</queue>");
// Give queue E a preemption timeout of one minute
out.println("<queue name=\"queueE\">");
@@ -194,6 +195,8 @@ public class TestAllocationFileLoaderSer
out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
// Set default limit of apps per user to 5
out.println("<userMaxAppsDefault>5</userMaxAppsDefault>");
+ // Set default limit of AMResourceShare to 0.5f
+ out.println("<queueMaxAMShareDefault>0.5f</queueMaxAMShareDefault>");
// Give user1 a limit of 10 jobs
out.println("<user name=\"user1\">");
out.println("<maxRunningApps>10</maxRunningApps>");
@@ -240,6 +243,13 @@ public class TestAllocationFileLoaderSer
assertEquals(10, queueConf.getUserMaxApps("user1"));
assertEquals(5, queueConf.getUserMaxApps("user2"));
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root." + YarnConfiguration.DEFAULT_QUEUE_NAME), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueA"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueB"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueC"), 0.01);
+ assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01);
+ assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01);
+
// Root should get * ACL
assertEquals("*", queueConf.getQueueAcl("root",
QueueACL.ADMINISTER_QUEUE).getAclString());
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java Mon Jun 16 18:13:57 2014
@@ -48,6 +48,8 @@ public class TestFSLeafQueue {
ResourceManager resourceManager = new ResourceManager();
resourceManager.init(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
String queueName = "root.queue1";
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon Jun 16 18:13:57 2014
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
@@ -73,12 +72,12 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
@@ -120,6 +119,8 @@ public class TestFairScheduler extends F
// to initialize the master key
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
+
+ scheduler.setRMContext(resourceManager.getRMContext());
}
@After
@@ -133,12 +134,12 @@ public class TestFairScheduler extends F
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new FairScheduler();
+ FairScheduler scheduler = new FairScheduler();
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, null);
+ scheduler.serviceInit(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
@@ -152,7 +153,7 @@ public class TestFairScheduler extends F
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1);
try {
- scheduler.reinitialize(conf, null);
+ scheduler.serviceInit(conf);
fail("Exception is expected because the min vcores allocation is" +
" larger than the max vcores allocation.");
} catch (YarnRuntimeException e) {
@@ -184,6 +185,8 @@ public class TestFairScheduler extends F
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
128);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
Assert.assertEquals(true, scheduler.assignMultiple);
Assert.assertEquals(3, scheduler.maxAssign);
@@ -211,6 +214,7 @@ public class TestFairScheduler extends F
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
conf.setInt(
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+ fs.init(conf);
fs.reinitialize(conf, null);
Assert.assertEquals(256, fs.getMinimumResourceCapability().getMemory());
Assert.assertEquals(1, fs.getMinimumResourceCapability().getVirtualCores());
@@ -228,8 +232,9 @@ public class TestFairScheduler extends F
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
conf.setInt(
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
- fs.reinitialize(conf, null);
- Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory());
+ fs.init(conf);
+ fs.reinitialize(conf, null);
+ Assert.assertEquals(0, fs.getMinimumResourceCapability().getMemory());
Assert.assertEquals(0, fs.getMinimumResourceCapability().getVirtualCores());
Assert.assertEquals(512, fs.getIncrementResourceCapability().getMemory());
Assert.assertEquals(2, fs.getIncrementResourceCapability().getVirtualCores());
@@ -237,6 +242,8 @@ public class TestFairScheduler extends F
@Test
public void testAggregateCapacityTracking() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -262,6 +269,8 @@ public class TestFairScheduler extends F
@Test
public void testSimpleFairShareCalculation() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add one big node (only care about aggregate capacity)
@@ -289,6 +298,8 @@ public class TestFairScheduler extends F
@Test
public void testSimpleHierarchicalFairShareCalculation() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add one big node (only care about aggregate capacity)
@@ -322,6 +333,8 @@ public class TestFairScheduler extends F
@Test
public void testHierarchicalQueuesSimilarParents() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueManager = scheduler.getQueueManager();
@@ -346,6 +359,8 @@ public class TestFairScheduler extends F
@Test
public void testSchedulerRootQueueMetrics() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -385,6 +400,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testSimpleContainerAllocation() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -433,6 +450,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testSimpleContainerReservation() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -487,48 +506,27 @@ public class TestFairScheduler extends F
@Test
public void testUserAsDefaultQueue() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
- RMContext rmContext = resourceManager.getRMContext();
- Map<ApplicationId, RMApp> appsMap = rmContext.getRMApps();
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
- RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
- null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null, null);
- appsMap.put(appAttemptId.getApplicationId(), rmApp);
-
- AppAddedSchedulerEvent appAddedEvent =
- new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default",
- "user1");
- scheduler.handle(appAddedEvent);
- AppAttemptAddedSchedulerEvent attempAddedEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- scheduler.handle(attempAddedEvent);
+ createApplicationWithAMResource(appAttemptId, "default", "user1", null);
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getRunnableAppSchedulables().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
.getRunnableAppSchedulables().size());
- assertEquals("root.user1", rmApp.getQueue());
+ assertEquals("root.user1", resourceManager.getRMContext().getRMApps()
+ .get(appAttemptId.getApplicationId()).getQueue());
}
@Test
public void testNotUserAsDefaultQueue() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
- RMContext rmContext = resourceManager.getRMContext();
- Map<ApplicationId, RMApp> appsMap = rmContext.getRMApps();
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
- RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
- null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null, null);
- appsMap.put(appAttemptId.getApplicationId(), rmApp);
-
- AppAddedSchedulerEvent appAddedEvent =
- new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default",
- "user2");
- scheduler.handle(appAddedEvent);
- AppAttemptAddedSchedulerEvent attempAddedEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- scheduler.handle(attempAddedEvent);
+ createApplicationWithAMResource(appAttemptId, "default", "user2", null);
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
.getRunnableAppSchedulables().size());
assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
@@ -539,6 +537,8 @@ public class TestFairScheduler extends F
@Test
public void testEmptyQueueName() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// only default queue
@@ -559,8 +559,10 @@ public class TestFairScheduler extends F
@Test
public void testAssignToQueue() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
@@ -577,6 +579,8 @@ public class TestFairScheduler extends F
@Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
@@ -594,6 +598,8 @@ public class TestFairScheduler extends F
public void testQueuePlacementWithPolicy() throws Exception {
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
ApplicationAttemptId appId;
@@ -654,7 +660,9 @@ public class TestFairScheduler extends F
out.println("</queue>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add one big node (only care about aggregate capacity)
@@ -703,6 +711,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
@@ -735,6 +745,8 @@ public class TestFairScheduler extends F
RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
int capacity = 16 * 1024;
@@ -769,6 +781,8 @@ public class TestFairScheduler extends F
*/
@Test
public void testQueueDemandCalculation() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
ApplicationAttemptId id11 = createAppAttemptId(1, 1);
@@ -819,6 +833,8 @@ public class TestFairScheduler extends F
@Test
public void testAppAdditionAndRemoval() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
ApplicationAttemptId attemptId =createAppAttemptId(1, 1);
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default",
@@ -869,6 +885,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueManager = scheduler.getQueueManager();
@@ -901,7 +919,9 @@ public class TestFairScheduler extends F
out.println("</queue>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueManager = scheduler.getQueueManager();
@@ -928,6 +948,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add one big node (only care about aggregate capacity)
@@ -985,8 +1007,10 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
// Add one big node (only care about aggregate capacity)
RMNode node1 =
MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1,
@@ -1057,7 +1081,9 @@ public class TestFairScheduler extends F
out.println("</queue>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Create two nodes
@@ -1224,6 +1250,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Create four nodes
@@ -1321,6 +1349,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testMultipleContainersWaitingForReservation() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -1363,8 +1393,10 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
// Add a node
RMNode node1 =
MockNodes
@@ -1404,6 +1436,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testReservationWhileMultiplePriorities() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
@@ -1484,8 +1518,10 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
"norealuserhasthisname", 1);
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
@@ -1499,6 +1535,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testMultipleNodesSingleRackRequest() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 =
@@ -1548,6 +1586,8 @@ public class TestFairScheduler extends F
@Test (timeout = 5000)
public void testFifoWithinQueue() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 =
@@ -1592,6 +1632,8 @@ public class TestFairScheduler extends F
@Test(timeout = 3000)
public void testMaxAssign() throws Exception {
conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node =
@@ -1635,6 +1677,8 @@ public class TestFairScheduler extends F
*/
@Test(timeout = 5000)
public void testAssignContainer() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
final String user = "user1";
@@ -1718,9 +1762,11 @@ public class TestFairScheduler extends F
out.println("</queue>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
int appId = this.APP_ID++;
String user = "usernotallow";
String queue = "queue1";
@@ -1769,6 +1815,8 @@ public class TestFairScheduler extends F
@Test
public void testReservationThatDoesntFit() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 =
@@ -1797,6 +1845,8 @@ public class TestFairScheduler extends F
@Test
public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
@@ -1825,6 +1875,8 @@ public class TestFairScheduler extends F
@Test
public void testStrictLocality() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
@@ -1865,6 +1917,8 @@ public class TestFairScheduler extends F
@Test
public void testCancelStrictLocality() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
@@ -1915,6 +1969,8 @@ public class TestFairScheduler extends F
*/
@Test
public void testReservationsStrictLocality() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
@@ -1955,6 +2011,8 @@ public class TestFairScheduler extends F
@Test
public void testNoMoreCpuOnNode() throws IOException {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1),
@@ -1976,6 +2034,8 @@ public class TestFairScheduler extends F
@Test
public void testBasicDRFAssignment() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5));
@@ -2016,6 +2076,8 @@ public class TestFairScheduler extends F
*/
@Test
public void testBasicDRFWithQueues() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7),
@@ -2052,6 +2114,8 @@ public class TestFairScheduler extends F
@Test
public void testDRFHierarchicalQueues() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(12288, 12),
@@ -2120,7 +2184,9 @@ public class TestFairScheduler extends F
public void testHostPortNodeName() throws Exception {
conf.setBoolean(YarnConfiguration
.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
- scheduler.reinitialize(conf,
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf,
resourceManager.getRMContext());
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024),
1, "127.0.0.1", 1);
@@ -2200,9 +2266,11 @@ public class TestFairScheduler extends F
out.println("</user>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
// exceeds no limits
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1");
verifyAppRunnable(attId1, true);
@@ -2236,6 +2304,186 @@ public class TestFairScheduler extends F
}
@Test
+ public void testQueueMaxAMShare() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("<maxAMShare>0.2</maxAMShare>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(20480, 20),
+ 0, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.update();
+
+ FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true);
+ assertEquals("Queue queue1's fair share should be 10240",
+ 10240, queue1.getFairShare().getMemory());
+
+ Resource amResource1 = Resource.newInstance(1024, 1);
+ Resource amResource2 = Resource.newInstance(2048, 2);
+ Resource amResource3 = Resource.newInstance(1860, 2);
+ int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
+ // Exceeds no limits
+ ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(attId1, "queue1", "user1", amResource1);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application1's AM requests 1024 MB memory",
+ 1024, app1.getAMResource().getMemory());
+ assertEquals("Application1's AM should be running",
+ 1, app1.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 1024 MB memory",
+ 1024, queue1.getAmResourceUsage().getMemory());
+
+ // Exceeds no limits
+ ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
+ createApplicationWithAMResource(attId2, "queue1", "user1", amResource1);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application2's AM requests 1024 MB memory",
+ 1024, app2.getAMResource().getMemory());
+ assertEquals("Application2's AM should be running",
+ 1, app2.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Exceeds queue limit
+ ApplicationAttemptId attId3 = createAppAttemptId(3, 1);
+ createApplicationWithAMResource(attId3, "queue1", "user1", amResource1);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3);
+ FSSchedulerApp app3 = scheduler.getSchedulerApp(attId3);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application3's AM requests 1024 MB memory",
+ 1024, app3.getAMResource().getMemory());
+ assertEquals("Application3's AM should not be running",
+ 0, app3.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Still can run non-AM container
+ createSchedulingRequestExistingApplication(1024, 1, attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application1 should have two running containers",
+ 2, app1.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Remove app1, app3's AM should become running
+ AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
+ new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED, false);
+ scheduler.update();
+ scheduler.handle(appRemovedEvent1);
+ scheduler.handle(updateEvent);
+ assertEquals("Application1's AM should be finished",
+ 0, app1.getLiveContainers().size());
+ assertEquals("Application3's AM should be running",
+ 1, app3.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Exceeds queue limit
+ ApplicationAttemptId attId4 = createAppAttemptId(4, 1);
+ createApplicationWithAMResource(attId4, "queue1", "user1", amResource2);
+ createSchedulingRequestExistingApplication(2048, 2, amPriority, attId4);
+ FSSchedulerApp app4 = scheduler.getSchedulerApp(attId4);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application4's AM requests 2048 MB memory",
+ 2048, app4.getAMResource().getMemory());
+ assertEquals("Application4's AM should not be running",
+ 0, app4.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Exceeds queue limit
+ ApplicationAttemptId attId5 = createAppAttemptId(5, 1);
+ createApplicationWithAMResource(attId5, "queue1", "user1", amResource2);
+ createSchedulingRequestExistingApplication(2048, 2, amPriority, attId5);
+ FSSchedulerApp app5 = scheduler.getSchedulerApp(attId5);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application5's AM requests 2048 MB memory",
+ 2048, app5.getAMResource().getMemory());
+ assertEquals("Application5's AM should not be running",
+ 0, app5.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Remove un-running app doesn't affect others
+ AppAttemptRemovedSchedulerEvent appRemovedEvent4 =
+ new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.KILLED, false);
+ scheduler.handle(appRemovedEvent4);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application5's AM should not be running",
+ 0, app5.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Remove app2 and app3, app5's AM should become running
+ AppAttemptRemovedSchedulerEvent appRemovedEvent2 =
+ new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED, false);
+ AppAttemptRemovedSchedulerEvent appRemovedEvent3 =
+ new AppAttemptRemovedSchedulerEvent(attId3, RMAppAttemptState.FINISHED, false);
+ scheduler.handle(appRemovedEvent2);
+ scheduler.handle(appRemovedEvent3);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application2's AM should be finished",
+ 0, app2.getLiveContainers().size());
+ assertEquals("Application3's AM should be finished",
+ 0, app3.getLiveContainers().size());
+ assertEquals("Application5's AM should be running",
+ 1, app5.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Check amResource normalization
+ ApplicationAttemptId attId6 = createAppAttemptId(6, 1);
+ createApplicationWithAMResource(attId6, "queue1", "user1", amResource3);
+ createSchedulingRequestExistingApplication(1860, 2, amPriority, attId6);
+ FSSchedulerApp app6 = scheduler.getSchedulerApp(attId6);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application6's AM should not be running",
+ 0, app6.getLiveContainers().size());
+ assertEquals("Application6's AM requests 2048 MB memory",
+ 2048, app6.getAMResource().getMemory());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Remove all apps
+ AppAttemptRemovedSchedulerEvent appRemovedEvent5 =
+ new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.FINISHED, false);
+ AppAttemptRemovedSchedulerEvent appRemovedEvent6 =
+ new AppAttemptRemovedSchedulerEvent(attId6, RMAppAttemptState.FINISHED, false);
+ scheduler.handle(appRemovedEvent5);
+ scheduler.handle(appRemovedEvent6);
+ scheduler.update();
+ assertEquals("Queue1's AM resource usage should be 0",
+ 0, queue1.getAmResourceUsage().getMemory());
+ }
+
+ @Test
public void testMaxRunningAppsHierarchicalQueues() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
MockClock clock = new MockClock();
@@ -2254,9 +2502,11 @@ public class TestFairScheduler extends F
out.println("</queue>");
out.println("</allocations>");
out.close();
-
+
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
// exceeds no limits
ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1");
verifyAppRunnable(attId1, true);
@@ -2316,6 +2566,9 @@ public class TestFairScheduler extends F
Configuration conf = createConfiguration();
conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED,
true);
+ fs.setRMContext(resourceManager.getRMContext());
+ fs.init(conf);
+ fs.start();
fs.reinitialize(conf, resourceManager.getRMContext());
Assert.assertTrue("Continuous scheduling should be enabled.",
fs.isContinuousSchedulingEnabled());
@@ -2396,6 +2649,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueManager = scheduler.getQueueManager();
@@ -2439,6 +2694,8 @@ public class TestFairScheduler extends F
out.println("</allocations>");
out.close();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
List<QueuePlacementRule> rules = scheduler.allocConf.placementPolicy
@@ -2455,6 +2712,8 @@ public class TestFairScheduler extends F
@SuppressWarnings("resource")
@Test
public void testBlacklistNodes() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
final int GB = 1024;
@@ -2507,6 +2766,8 @@ public class TestFairScheduler extends F
@Test
public void testGetAppsInQueue() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
ApplicationAttemptId appAttId1 =
@@ -2552,8 +2813,10 @@ public class TestFairScheduler extends F
@Test
public void testMoveRunnableApp() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
@@ -2591,8 +2854,10 @@ public class TestFairScheduler extends F
@Test
public void testMoveNonRunnableApp() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
@@ -2611,8 +2876,10 @@ public class TestFairScheduler extends F
@Test
public void testMoveMakesAppRunnable() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
@@ -2638,8 +2905,10 @@ public class TestFairScheduler extends F
@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
QueueManager queueMgr = scheduler.getQueueManager();
queueMgr.getLeafQueue("queue2", true);
scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
@@ -2652,8 +2921,10 @@ public class TestFairScheduler extends F
@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxResourcesConstraints() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
-
+
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
queueMgr.getLeafQueue("queue2", true);
@@ -2675,6 +2946,8 @@ public class TestFairScheduler extends F
@Test (expected = YarnException.class)
public void testMoveToNonexistentQueue() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.getQueueManager().getLeafQueue("queue1", true);
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java Mon Jun 16 18:13:57 2014
@@ -51,6 +51,8 @@ public class TestFairSchedulerEventLog {
resourceManager = new ResourceManager();
resourceManager.init(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
}
@@ -69,7 +71,13 @@ public class TestFairSchedulerEventLog {
public void tearDown() {
logFile.delete();
logFile.getParentFile().delete(); // fairscheduler/
- scheduler = null;
- resourceManager = null;
+ if (scheduler != null) {
+ scheduler.stop();
+ scheduler = null;
+ }
+ if (resourceManager != null) {
+ resourceManager.stop();
+ resourceManager = null;
+ }
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java Mon Jun 16 18:13:57 2014
@@ -146,7 +146,7 @@ public class TestFairSchedulerPreemption
// Create node with 4GB memory and 4 vcores
registerNodeAndSubmitApp(4 * 1024, 4, 2, 1024);
- // Verify submitting another request doesn't trigger preemption
+ // Verify submitting another request triggers preemption
createSchedulingRequest(1024, "queueB", "user1", 1, 1);
scheduler.update();
clock.tick(6);
@@ -171,5 +171,21 @@ public class TestFairSchedulerPreemption
scheduler.preemptTasksIfNecessary();
assertEquals("preemptResources() should not have been called", -1,
((StubbedFairScheduler) scheduler).lastPreemptMemory);
+
+ resourceManager.stop();
+
+ startResourceManager(0.7f);
+ // Create node with 4GB memory and 4 vcores
+ registerNodeAndSubmitApp(4 * 1024, 4, 3, 1024);
+
+ // Verify submitting another request triggers preemption
+ createSchedulingRequest(1024, "queueB", "user1", 1, 1);
+ scheduler.update();
+ clock.tick(6);
+
+ ((StubbedFairScheduler) scheduler).resetLastPreemptResources();
+ scheduler.preemptTasksIfNecessary();
+ assertEquals("preemptResources() should have been called", 1024,
+ ((StubbedFairScheduler) scheduler).lastPreemptMemory);
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Mon Jun 16 18:13:57 2014
@@ -145,9 +145,13 @@ public class TestFifoScheduler {
RMContext rmContext = new RMContextImpl(dispatcher, null,
null, null, null, null, null, null, null, writer);
- FifoScheduler schedular = new FifoScheduler();
- schedular.reinitialize(new Configuration(), rmContext);
- QueueMetrics metrics = schedular.getRootQueueMetrics();
+ FifoScheduler scheduler = new FifoScheduler();
+ Configuration conf = new Configuration();
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, rmContext);
+ QueueMetrics metrics = scheduler.getRootQueueMetrics();
int beforeAppsSubmitted = metrics.getAppsSubmitted();
ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
@@ -155,18 +159,19 @@ public class TestFifoScheduler {
appId, 1);
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
- schedular.handle(appEvent);
+ scheduler.handle(appEvent);
SchedulerEvent attemptEvent =
new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- schedular.handle(attemptEvent);
+ scheduler.handle(attemptEvent);
appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
SchedulerEvent attemptEvent2 =
new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- schedular.handle(attemptEvent2);
+ scheduler.handle(attemptEvent2);
int afterAppsSubmitted = metrics.getAppsSubmitted();
Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
+ scheduler.stop();
}
@Test(timeout=2000)
@@ -184,6 +189,9 @@ public class TestFifoScheduler {
null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
FifoScheduler scheduler = new FifoScheduler();
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.newNodeInfo(1,
@@ -232,6 +240,7 @@ public class TestFifoScheduler {
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
Assert.assertEquals(3, info.getLiveContainers().size());
+ scheduler.stop();
}
@Test(timeout=2000)
@@ -254,6 +263,9 @@ public class TestFifoScheduler {
return nodes;
}
};
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.newNodeInfo(1,
Resources.createResource(2048, 4), 1, "127.0.0.1");
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Mon Jun 16 18:13:57 2014
@@ -203,10 +203,11 @@ public class TestRMWebApp {
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
- cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ cs.setRMContext(new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null));
+ cs.init(conf);
return cs;
}
@@ -269,19 +270,21 @@ public class TestRMWebApp {
ResourceManager rm = mock(ResourceManager.class);
RMContext rmContext = mockRMContext(apps, racks, nodes,
mbsPerNode);
- ResourceScheduler rs = mockFifoScheduler();
+ ResourceScheduler rs = mockFifoScheduler(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
return rm;
}
- public static FifoScheduler mockFifoScheduler() throws Exception {
+ public static FifoScheduler mockFifoScheduler(RMContext rmContext)
+ throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupFifoQueueConfiguration(conf);
FifoScheduler fs = new FifoScheduler();
fs.setConf(new YarnConfiguration());
- fs.reinitialize(conf, null);
+ fs.setRMContext(rmContext);
+ fs.init(conf);
return fs;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java Mon Jun 16 18:13:57 2014
@@ -25,9 +25,12 @@ import org.apache.hadoop.service.Service
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.net.InetSocketAddress;
+
public class TestWebAppProxyServer {
private WebAppProxyServer webAppProxy = null;
private final String proxyAddress = "0.0.0.0:8888";
@@ -56,4 +59,14 @@ public class TestWebAppProxyServer {
}
assertEquals(STATE.STARTED, webAppProxy.getServiceState());
}
+
+ @Test
+ public void testBindAddress() {
+ YarnConfiguration conf = new YarnConfiguration();
+
+ InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf);
+ Assert.assertEquals("Web Proxy default bind address port is incorrect",
+ YarnConfiguration.DEFAULT_PROXY_PORT,
+ defaultBindAddress.getPort());
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon Jun 16 18:13:57 2014
@@ -237,6 +237,11 @@ Allocation file format
* maxRunningApps: limit the number of apps from the queue to run at once
+ * maxAMShare: limit the fraction of the queue's fair share that can be used
+ to run application masters. This property can only be used for leaf queues.
+ Default value is 1.0f, which means AMs in the leaf queue can take up to 100%
+ of both the memory and CPU fair share.
+
* weight: to share the cluster non-proportionally with other queues. Weights
default to 1, and a queue with weight 2 should receive approximately twice
as many resources as a queue with the default weight.
@@ -279,6 +284,9 @@ Allocation file format
* <<A queueMaxAppsDefault element>>, which sets the default running app limit
for queues; overriden by maxRunningApps element in each queue.
+ * <<A queueMaxAMShareDefault element>>, which sets the default AM resource
+ limit for queue; overriden by maxAMShare element in each queue.
+
* <<A defaultQueueSchedulingPolicy element>>, which sets the default scheduling
policy for queues; overriden by the schedulingPolicy element in each queue
if specified. Defaults to "fair".
@@ -328,6 +336,7 @@ Allocation file format
<minResources>10000 mb,0vcores</minResources>
<maxResources>90000 mb,0vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
+ <maxAMShare>0.1</maxAMShare>
<weight>2.0</weight>
<schedulingPolicy>fair</schedulingPolicy>
<queue name="sample_sub_queue">
@@ -336,6 +345,8 @@ Allocation file format
</queue>
</queue>
+ <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
+
<!â- Queue âsecondary_group_queueâ is a parent queue and may have
user queues under it -â>
<queue name=âsecondary_group_queueâ type=âparentâ>
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1602947&r1=1602946&r2=1602947&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Mon Jun 16 18:13:57 2014
@@ -1564,6 +1564,287 @@ _01_000001</amContainerLogs>
</app>
+---+
+* Cluster Application State API
+
+ With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response.
+
+ Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response.
+
+** URI
+
+-----
+ * http://<rm http address:port>/ws/v1/cluster/apps/{appid}/state
+-----
+
+** HTTP Operations Supported
+
+------
+ * GET
+ * PUT
+------
+
+** Query Parameters Supported
+
+------
+ None
+------
+
+** Elements of <appstate> object
+
+ When you make a request for the state of an app, the information returned has the following fields
+
+*---------------+--------------+-------------------------------+
+|| Item || Data Type || Description |
+*---------------+--------------+-------------------------------+
+| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" |
+*---------------+--------------+--------------------------------+
+
+
+** Response Examples
+
+ <<JSON responses>>
+
+ HTTP Request
+
+-----
+ GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+-----
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"ACCEPTED"
+}
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 202 Accepted
+Content-Type: application/json
+Transfer-Encoding: chunked
+Location: http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"ACCEPTED"
+}
++---+
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ <<XML responses>>
+
+ HTTP Request
+
+-----
+ GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+-----
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 99
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>ACCEPTED</state>
+</appstate>
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 202 Accepted
+Content-Type: application/json
+Content-Length: 794
+Location: http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>ACCEPTED</state>
+</appstate>
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 917
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ <<Unauthorized Error Response>>
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 403 Unauthorized
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+
+ <<Bad Request Error Response>>
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>RUNNING</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 400
+Content-Length: 295
+Content-Type: application/xml
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<RemoteException>
+ <exception>BadRequestException</exception>
+ <message>java.lang.Exception: Only 'KILLED' is allowed as a target state.</message>
+ <javaClassName>org.apache.hadoop.yarn.webapp.BadRequestException</javaClassName>
+</RemoteException>
++---+
+
* Cluster Application Attempts API
With the application attempts API, you can obtain a collection of resources that represent an application attempt. When you run a GET operation on this resource, you obtain a collection of App Attempt Objects.