You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/10/30 23:22:36 UTC
svn commit: r1537330 [10/12] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/jav...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Oct 30 22:21:59 2013
@@ -25,18 +25,25 @@ import static org.junit.Assert.assertNul
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.spy;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -47,7 +54,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
@@ -79,13 +85,19 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(value = Parameterized.class)
public class TestRMAppAttemptTransitions {
private static final Log LOG =
@@ -93,8 +105,9 @@ public class TestRMAppAttemptTransitions
private static final String EMPTY_DIAGNOSTICS = "";
private static final String RM_WEBAPP_ADDR =
- YarnConfiguration.getRMWebAppHostAndPort(new Configuration());
+ WebAppUtils.getResolvedRMWebAppURLWithoutScheme(new Configuration());
+ private boolean isSecurityEnabled;
private RMContext rmContext;
private YarnScheduler scheduler;
private ApplicationMasterService masterService;
@@ -162,8 +175,26 @@ public class TestRMAppAttemptTransitions
private ApplicationSubmissionContext submissionContext = null;
private boolean unmanagedAM;
+ @Parameterized.Parameters
+ public static Collection<Object[]> getTestParameters() {
+ return Arrays.asList(new Object[][] {
+ { Boolean.FALSE },
+ { Boolean.TRUE }
+ });
+ }
+
+ public TestRMAppAttemptTransitions(Boolean isSecurityEnabled) {
+ this.isSecurityEnabled = isSecurityEnabled;
+ }
+
@Before
public void setUp() throws Exception {
+ AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
+ if (isSecurityEnabled) {
+ authMethod = AuthenticationMethod.KERBEROS;
+ }
+ SecurityUtil.setAuthenticationMethod(authMethod, conf);
+ UserGroupInformation.setConfiguration(conf);
InlineDispatcher rmDispatcher = new InlineDispatcher();
ContainerAllocationExpirer containerAllocationExpirer =
@@ -235,8 +266,21 @@ public class TestRMAppAttemptTransitions
private String getProxyUrl(RMAppAttempt appAttempt) {
- return pjoin(RM_WEBAPP_ADDR, "proxy",
- appAttempt.getAppAttemptId().getApplicationId(), "");
+ String url = null;
+ try {
+ URI trackingUri =
+ StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
+ ProxyUriUtils
+ .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+ URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+ appAttempt.getAppAttemptId().getApplicationId());
+ url = result.toASCIIString();
+ } catch (URISyntaxException ex) {
+ Assert.fail();
+ }
+ return url;
}
/**
@@ -268,9 +312,13 @@ public class TestRMAppAttemptTransitions
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
if (UserGroupInformation.isSecurityEnabled()) {
- verify(clientToAMTokenManager).registerApplication(
+ verify(clientToAMTokenManager).createMasterKey(
applicationAttempt.getAppAttemptId());
+ // can't create ClientToken as at this time ClientTokenMasterKey has
+ // not been registered in the SecretManager
+ assertNull(applicationAttempt.createClientToken("some client"));
}
+ assertNull(applicationAttempt.createClientToken(null));
assertNotNull(applicationAttempt.getAMRMToken());
// Check events
verify(masterService).
@@ -400,7 +448,11 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.LAUNCHED,
applicationAttempt.getAppAttemptState());
assertEquals(container, applicationAttempt.getMasterContainer());
-
+ if (UserGroupInformation.isSecurityEnabled()) {
+ // ClientTokenMasterKey has been registered in SecretManager, it's able to
+ // create ClientToken now
+ assertNotNull(applicationAttempt.createClientToken("some client"));
+ }
// TODO - need to add more checks relevant to this state
}
@@ -408,16 +460,19 @@ public class TestRMAppAttemptTransitions
* {@link RMAppAttemptState#RUNNING}
*/
private void testAppAttemptRunningState(Container container,
- String host, int rpcPort, String trackingUrl) {
+ String host, int rpcPort, String trackingUrl, boolean unmanagedAM) {
assertEquals(RMAppAttemptState.RUNNING,
applicationAttempt.getAppAttemptState());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals(getProxyUrl(applicationAttempt),
- applicationAttempt.getTrackingUrl());
-
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ if (unmanagedAM) {
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
+ } else {
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
+ }
// TODO - need to add more checks relevant to this state
}
@@ -431,7 +486,7 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHING,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
@@ -446,13 +501,18 @@ public class TestRMAppAttemptTransitions
FinalApplicationStatus finalStatus,
String trackingUrl,
String diagnostics,
- int finishedContainerCount) {
+ int finishedContainerCount, boolean unmanagedAM) {
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals(getProxyUrl(applicationAttempt),
- applicationAttempt.getTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ if (unmanagedAM) {
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
+
+ } else {
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
+ }
assertEquals(finishedContainerCount, applicationAttempt
.getJustFinishedContainers().size());
assertEquals(container, applicationAttempt.getMasterContainer());
@@ -525,6 +585,11 @@ public class TestRMAppAttemptTransitions
}
private void launchApplicationAttempt(Container container) {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ // Before LAUNCHED state, can't create ClientToken as at this time
+ // ClientTokenMasterKey has not been registered in the SecretManager
+ assertNull(applicationAttempt.createClientToken("some client"));
+ }
applicationAttempt.handle(
new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.LAUNCHED));
@@ -535,13 +600,14 @@ public class TestRMAppAttemptTransitions
private void runApplicationAttempt(Container container,
String host,
int rpcPort,
- String trackingUrl) {
+ String trackingUrl, boolean unmanagedAM) {
applicationAttempt.handle(
new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(),
host, rpcPort, trackingUrl));
- testAppAttemptRunningState(container, host, rpcPort, trackingUrl);
+ testAppAttemptRunningState(container, host, rpcPort, trackingUrl,
+ unmanagedAM);
}
private void unregisterApplicationAttempt(Container container,
@@ -555,9 +621,7 @@ public class TestRMAppAttemptTransitions
trackingUrl, diagnostics);
}
-
- @Test
- public void testUnmanagedAMSuccess() {
+ private void testUnmanagedAMSuccess(String url) {
unmanagedAM = true;
when(submissionContext.getUnmanagedAM()).thenReturn(true);
// submit AM and check it goes to LAUNCHED state
@@ -567,7 +631,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId());
// launch AM
- runApplicationAttempt(null, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(null, "host", 8042, url, true);
// complete a container
applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@@ -575,13 +639,13 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
// complete AM
- String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
- applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
+ applicationAttempt.getAppAttemptId(), url, finalStatus,
diagnostics));
- testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1);
+ testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
+ true);
}
@Test
@@ -690,7 +754,7 @@ public class TestRMAppAttemptTransitions
public void testRunningToFailed() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
String containerDiagMsg = "some error";
int exitCode = 123;
ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
@@ -713,7 +777,7 @@ public class TestRMAppAttemptTransitions
public void testRunningToKilled() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
applicationAttempt.handle(
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
@@ -751,7 +815,7 @@ public class TestRMAppAttemptTransitions
public void testRunningExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FAILED,
@@ -769,25 +833,55 @@ public class TestRMAppAttemptTransitions
public void testUnregisterToKilledFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
unregisterApplicationAttempt(amContainer,
FinalApplicationStatus.KILLED, "newtrackingurl",
"Killed by user");
}
+ @Test
+ public void testTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("oldTrackingUrl");
+ }
+
+ @Test
+ public void testEmptyTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("");
+ }
+
+ @Test
+ public void testNullTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess(null);
+ }
+
+ @Test
+ public void testManagedAMWithTrackingUrl() {
+ testTrackingUrlManagedAM("theTrackingUrl");
+ }
+
+ @Test
+ public void testManagedAMWithEmptyTrackingUrl() {
+ testTrackingUrlManagedAM("");
+ }
@Test
- public void testNoTrackingUrl() {
+ public void testManagedAMWithNullTrackingUrl() {
+ testTrackingUrlManagedAM(null);
+ }
+
+ private void testTrackingUrlManagedAM(String url) {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "");
+ runApplicationAttempt(amContainer, "host", 8042, url, false);
+ unregisterApplicationAttempt(amContainer,
+ FinalApplicationStatus.SUCCEEDED, url, "Successful");
}
@Test
public void testUnregisterToSuccessfulFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
unregisterApplicationAttempt(amContainer,
FinalApplicationStatus.SUCCEEDED, "mytrackingurl", "Successful");
}
@@ -796,7 +890,7 @@ public class TestRMAppAttemptTransitions
public void testFinishingKill() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.FAILED;
String trackingUrl = "newtrackingurl";
String diagnostics = "Job failed";
@@ -814,7 +908,7 @@ public class TestRMAppAttemptTransitions
public void testFinishingExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -825,14 +919,14 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.EXPIRE));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0);
+ diagnostics, 0, false);
}
@Test
public void testFinishingToFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -854,7 +948,7 @@ public class TestRMAppAttemptTransitions
public void testSuccessfulFinishingToFinished() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -866,13 +960,24 @@ public class TestRMAppAttemptTransitions
BuilderUtils.newContainerStatus(amContainer.getId(),
ContainerState.COMPLETE, "", 0)));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0);
+ diagnostics, 0, false);
}
private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId);
if (UserGroupInformation.isSecurityEnabled()) {
verify(clientToAMTokenManager, times(count)).unRegisterApplication(appAttemptId);
+ if (count > 0) {
+ assertNull(applicationAttempt.createClientToken("client"));
+ }
+ }
+ }
+
+ private void verifyUrl(String url1, String url2) {
+ if (url1 == null || url1.trim().isEmpty()) {
+ assertEquals("N/A", url2);
+ } else {
+ assertEquals(url1, url2);
}
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Wed Oct 30 22:21:59 2013
@@ -41,6 +41,10 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
@@ -358,4 +362,20 @@ public class TestSchedulerUtils {
Priority low = Priority.newInstance(2);
assertTrue(high.compareTo(low) > 0);
}
+
+ @Test
+ public void testCreateAbnormalContainerStatus() {
+ ContainerStatus cd = SchedulerUtils.createAbnormalContainerStatus(
+ ContainerId.newInstance(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
+ Assert.assertEquals(ContainerExitStatus.ABORTED, cd.getExitStatus());
+ }
+
+ @Test
+ public void testCreatePreemptedContainerStatus() {
+ ContainerStatus cd = SchedulerUtils.createPreemptedContainerStatus(
+ ContainerId.newInstance(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
+ Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus());
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Wed Oct 30 22:21:59 2013
@@ -513,7 +513,7 @@ public class TestApplicationLimits {
app_0_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_0.updateResourceRequests(app_0_0_requests, null, null);
+ app_0_0.updateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.assignContainers(clusterResource, node_0);
@@ -532,7 +532,7 @@ public class TestApplicationLimits {
app_0_1_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_1.updateResourceRequests(app_0_1_requests, null, null);
+ app_0_1.updateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
@@ -551,7 +551,7 @@ public class TestApplicationLimits {
app_1_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_1_0.updateResourceRequests(app_1_0_requests, null, null);
+ app_1_0.updateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Wed Oct 30 22:21:59 2013
@@ -19,23 +19,30 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -44,19 +51,29 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -95,6 +112,8 @@ public class TestCapacityScheduler {
conf.setClass(YarnConfiguration.RM_SCHEDULER,
CapacityScheduler.class, ResourceScheduler.class);
resourceManager.init(conf);
+ resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
+ resourceManager.getRMNMTokenSecretManager().rollMasterKey();
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
}
@@ -140,13 +159,18 @@ public class TestCapacityScheduler {
registerNode(String hostName, int containerManagerPort, int httpPort,
String rackName, Resource capability)
throws IOException, YarnException {
- return new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
- hostName, containerManagerPort, httpPort, rackName, capability,
- resourceManager.getResourceTrackerService(), resourceManager
- .getRMContext());
- }
+ org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm =
+ new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
+ hostName, containerManagerPort, httpPort, rackName, capability,
+ resourceManager);
+ NodeAddedSchedulerEvent nodeAddEvent1 =
+ new NodeAddedSchedulerEvent(resourceManager.getRMContext()
+ .getRMNodes().get(nm.getNodeId()));
+ resourceManager.getResourceScheduler().handle(nodeAddEvent1);
+ return nm;
+ }
-// @Test
+ @Test
public void testCapacityScheduler() throws Exception {
LOG.info("--- START: testCapacityScheduler ---");
@@ -156,14 +180,12 @@ public class TestCapacityScheduler {
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
Resources.createResource(4 * GB, 1));
- nm_0.heartbeat();
// Register node2
String host_1 = "host_1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
Resources.createResource(2 * GB, 1));
- nm_1.heartbeat();
// ResourceRequest priorities
Priority priority_0 =
@@ -211,9 +233,13 @@ public class TestCapacityScheduler {
// Send a heartbeat to kick the tires on the Scheduler
LOG.info("Kick!");
- nm_0.heartbeat(); // task_0_0 and task_1_0 allocated, used=4G
- nm_1.heartbeat(); // nothing allocated
-
+
+ // task_0_0 and task_1_0 allocated, used=4G
+ nodeUpdate(nm_0);
+
+ // nothing allocated
+ nodeUpdate(nm_1);
+
// Get allocations from the scheduler
application_0.schedule(); // task_0_0
checkApplicationResourceUsage(1 * GB, application_0);
@@ -221,9 +247,6 @@ public class TestCapacityScheduler {
application_1.schedule(); // task_1_0
checkApplicationResourceUsage(3 * GB, application_1);
- nm_0.heartbeat();
- nm_1.heartbeat();
-
checkNodeResourceUsage(4*GB, nm_0); // task_0_0 (1G) and task_1_0 (3G)
checkNodeResourceUsage(0*GB, nm_1); // no tasks, 2G available
@@ -243,10 +266,12 @@ public class TestCapacityScheduler {
// Send a heartbeat to kick the tires on the Scheduler
LOG.info("Sending hb from " + nm_0.getHostName());
- nm_0.heartbeat(); // nothing new, used=4G
+ // nothing new, used=4G
+ nodeUpdate(nm_0);
LOG.info("Sending hb from " + nm_1.getHostName());
- nm_1.heartbeat(); // task_0_3, used=2G
+ // task_0_1 is prefer as locality, used=2G
+ nodeUpdate(nm_1);
// Get allocations from the scheduler
LOG.info("Trying to allocate...");
@@ -256,13 +281,22 @@ public class TestCapacityScheduler {
application_1.schedule();
checkApplicationResourceUsage(5 * GB, application_1);
- nm_0.heartbeat();
- nm_1.heartbeat();
+ nodeUpdate(nm_0);
+ nodeUpdate(nm_1);
+
checkNodeResourceUsage(4*GB, nm_0);
checkNodeResourceUsage(2*GB, nm_1);
LOG.info("--- END: testCapacityScheduler ---");
}
+
+ private void nodeUpdate(
+ org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm) {
+ RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId());
+ // Send a heartbeat to kick the tires on the Scheduler
+ NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
+ resourceManager.getResourceScheduler().handle(nodeUpdate);
+ }
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
@@ -502,8 +536,41 @@ public class TestCapacityScheduler {
}
return result;
}
-
-
+
+ @SuppressWarnings("resource")
+ @Test
+ public void testBlackListNodes() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ String host = "127.0.0.1";
+ RMNode node =
+ MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
+ cs.handle(new NodeAddedSchedulerEvent(node));
+
+ ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default",
+ "user");
+ cs.handle(event);
+
+ // Verify the blacklist can be updated independent of requesting containers
+ cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
+ Collections.<ContainerId>emptyList(),
+ Collections.singletonList(host), null);
+ Assert.assertTrue(cs.getApplication(appAttemptId).isBlacklisted(host));
+ cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
+ Collections.<ContainerId>emptyList(), null,
+ Collections.singletonList(host));
+ Assert.assertFalse(cs.getApplication(appAttemptId).isBlacklisted(host));
+ rm.stop();
+ }
+
@Test (timeout = 5000)
public void testApplicationComparator()
{
@@ -525,4 +592,63 @@ public class TestCapacityScheduler {
assertTrue(appComparator.compare(app2, app3) < 0);
}
+ @Test
+ public void testConcurrentAccessOnApplications() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ verifyConcurrentAccessOnApplications(
+ cs.applications, FiCaSchedulerApp.class);
+ }
+
+ public static <T extends SchedulerApplication>
+ void verifyConcurrentAccessOnApplications(
+ final Map<ApplicationAttemptId, T> applications, Class<T> clazz)
+ throws Exception {
+ final int size = 10000;
+ final ApplicationId appId = ApplicationId.newInstance(0, 0);
+ final Constructor<T> ctor = clazz.getDeclaredConstructor(
+ ApplicationAttemptId.class, String.class, Queue.class,
+ ActiveUsersManager.class, RMContext.class);
+
+ ApplicationAttemptId appAttemptId0
+ = ApplicationAttemptId.newInstance(appId, 0);
+ applications.put(appAttemptId0, ctor.newInstance(
+ appAttemptId0, null, mock(Queue.class), null, null));
+ assertNotNull(applications.get(appAttemptId0));
+
+ // Imitating the thread of scheduler that will add and remove apps
+ final AtomicBoolean finished = new AtomicBoolean(false);
+ final AtomicBoolean failed = new AtomicBoolean(false);
+ Thread t = new Thread() {
+
+ @Override
+ public void run() {
+ for (int i = 1; i <= size; ++i) {
+ ApplicationAttemptId appAttemptId
+ = ApplicationAttemptId.newInstance(appId, i);
+ try {
+ applications.put(appAttemptId, ctor.newInstance(
+ appAttemptId, null, mock(Queue.class), null, null));
+ } catch (Exception e) {
+ failed.set(true);
+ finished.set(true);
+ return;
+ }
+ }
+ for (int i = 1; i <= size; ++i) {
+ ApplicationAttemptId appAttemptId
+ = ApplicationAttemptId.newInstance(appId, i);
+ applications.remove(appAttemptId);
+ }
+ finished.set(true);
+ }
+ };
+ t.start();
+
+ // Imitating the thread of rmappattempt that will get the app
+ while (!finished.get()) {
+ assertNotNull(applications.get(appAttemptId0));
+ }
+ assertFalse(failed.get());
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Wed Oct 30 22:21:59 2013
@@ -151,7 +151,8 @@ public class TestLeafQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot});
conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
+ conf.setAcl(CapacitySchedulerConfiguration.ROOT,
+ QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot;
conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E});
@@ -282,8 +283,9 @@ public class TestLeafQueue {
// Setup some nodes
String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8*GB);
-
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 8*GB);
+
final int numNodes = 1;
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
@@ -293,13 +295,15 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Start testing...
// Only 1 container
a.assignContainers(clusterResource, node_0);
- assertEquals(6*GB, a.getMetrics().getAvailableMB());
+ assertEquals(
+ (int)(node_0.getTotalResource().getMemory() * a.getCapacity()) - (1*GB),
+ a.getMetrics().getAvailableMB());
}
@Test
@@ -404,8 +408,9 @@ public class TestLeafQueue {
// Setup some nodes
String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8*GB);
-
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 8*GB);
+
final int numNodes = 1;
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
@@ -415,11 +420,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Start testing...
@@ -492,12 +497,14 @@ public class TestLeafQueue {
a.completedContainer(clusterResource, app_1, node_0, rmContainer,
null, RMContainerEventType.KILL, null);
}
+
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0*GB, a.getMetrics().getAllocatedMB());
- assertEquals(1*GB, a.getMetrics().getAvailableMB());
+ assertEquals((int)(a.getCapacity() * node_0.getTotalResource().getMemory()),
+ a.getMetrics().getAvailableMB());
}
@Test
@@ -548,11 +555,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
/**
* Start testing...
@@ -573,7 +580,7 @@ public class TestLeafQueue {
// Pre MAPREDUCE-3732 this test should fail without this block too
// app_2.updateResourceRequests(Collections.singletonList(
// TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority,
-// recordFactory)));
+// recordFactory)));
// 1 container to user_0
a.assignContainers(clusterResource, node_0);
@@ -641,11 +648,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
/**
* Start testing...
@@ -680,7 +687,7 @@ public class TestLeafQueue {
a.setMaxCapacity(.1f);
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
assertEquals(2, a.getActiveUsersManager().getNumActiveUsers());
// No more to user_0 since he is already over user-limit
@@ -697,7 +704,7 @@ public class TestLeafQueue {
LOG.info("here");
app_1.updateResourceRequests(Collections.singletonList( // unset
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
assertEquals(1, a.getActiveUsersManager().getNumActiveUsers());
a.assignContainers(clusterResource, node_1);
assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap
@@ -758,11 +765,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
/**
* Start testing...
@@ -792,11 +799,11 @@ public class TestLeafQueue {
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_3.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Now allocations should goto app_2 since
// user_0 is at limit inspite of high user-limit-factor
@@ -920,11 +927,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Start testing...
@@ -1024,7 +1031,7 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Setup app_1 to request a 4GB container on host_0 and
// another 4GB container anywhere.
@@ -1036,7 +1043,7 @@ public class TestLeafQueue {
true, priority, recordFactory));
appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2,
true, priority, recordFactory));
- app_1.updateResourceRequests(appRequests_1, null, null);
+ app_1.updateResourceRequests(appRequests_1);
// Start testing...
@@ -1131,11 +1138,11 @@ public class TestLeafQueue {
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
- priority, recordFactory)), null, null);
+ priority, recordFactory)));
// Start testing...
@@ -1260,7 +1267,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
// Start testing...
CSAssignment assignment = null;
@@ -1325,7 +1332,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
assertEquals(2, app_0.getTotalRequiredResources(priority));
String host_3 = "127.0.0.4"; // on rack_1
@@ -1416,7 +1423,7 @@ public class TestLeafQueue {
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1,
true, priority_2, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
// Start testing...
@@ -1531,7 +1538,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
// Start testing...
@@ -1540,7 +1547,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
// NODE_LOCAL - node_0_1
a.assignContainers(clusterResource, node_0_0);
@@ -1563,7 +1570,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
// No allocation on node_0_1 even though it's node/rack local since
// required(rack_1) == 0
@@ -1808,8 +1815,8 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0,
- Collections.singletonList(host_0_0), null);
+ app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateBlacklist(Collections.singletonList(host_0_0), null);
app_0_requests_0.clear();
//
@@ -1849,8 +1856,8 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(rack_1, 1*GB, 1,
true, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0,
- Collections.singletonList(host_1_1), null);
+ app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateBlacklist(Collections.singletonList(host_1_1), null);
app_0_requests_0.clear();
// resourceName: <priority, memory, #containers, relaxLocality>
@@ -1876,7 +1883,8 @@ public class TestLeafQueue {
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
// Now, remove node_1_1 from blacklist, but add rack_1 to blacklist
- app_0.updateResourceRequests(app_0_requests_0,
+ app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateBlacklist(
Collections.singletonList(rack_1), Collections.singletonList(host_1_1));
app_0_requests_0.clear();
@@ -1903,8 +1911,8 @@ public class TestLeafQueue {
assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0
// Now remove rack_1 from blacklist
- app_0.updateResourceRequests(app_0_requests_0,
- null, Collections.singletonList(rack_1));
+ app_0.updateResourceRequests(app_0_requests_0);
+ app_0.updateBlacklist(null, Collections.singletonList(rack_1));
app_0_requests_0.clear();
// resourceName: <priority, memory, #containers, relaxLocality>
@@ -1936,7 +1944,7 @@ public class TestLeafQueue {
app_0_requests_0.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one
false, priority, recordFactory));
- app_0.updateResourceRequests(app_0_requests_0, null, null);
+ app_0.updateResourceRequests(app_0_requests_0);
app_0_requests_0.clear();
// resourceName: <priority, memory, #containers, relaxLocality>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java Wed Oct 30 22:21:59 2013
@@ -25,11 +25,25 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.util.Clock;
import org.junit.Test;
import org.mockito.Mockito;
public class TestFSSchedulerApp {
+ private class MockClock implements Clock {
+ private long time = 0;
+ @Override
+ public long getTime() {
+ return time;
+ }
+
+ public void tick(int seconds) {
+ time = time + seconds * 1000;
+ }
+
+ }
+
private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
ApplicationId appIdImpl = ApplicationId.newInstance(0, appId);
ApplicationAttemptId attId =
@@ -94,6 +108,63 @@ public class TestFSSchedulerApp {
}
@Test
+ public void testDelaySchedulingForContinuousScheduling()
+ throws InterruptedException {
+ Queue queue = Mockito.mock(Queue.class);
+ Priority prio = Mockito.mock(Priority.class);
+ Mockito.when(prio.getPriority()).thenReturn(1);
+
+ MockClock clock = new MockClock();
+
+ long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds
+ long rackLocalityDelayMs = 6 * 1000L; // 6 seconds
+
+ ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
+ FSSchedulerApp schedulerApp =
+ new FSSchedulerApp(applicationAttemptId, "user1", queue,
+ null, null);
+ AppSchedulable appSchedulable = Mockito.mock(AppSchedulable.class);
+ long startTime = clock.getTime();
+ Mockito.when(appSchedulable.getStartTime()).thenReturn(startTime);
+ schedulerApp.setAppSchedulable(appSchedulable);
+
+ // Default level should be node-local
+ assertEquals(NodeType.NODE_LOCAL,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+
+ // after 4 seconds should remain node local
+ clock.tick(4);
+ assertEquals(NodeType.NODE_LOCAL,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+
+ // after 6 seconds should switch to rack local
+ clock.tick(2);
+ assertEquals(NodeType.RACK_LOCAL,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+
+ // manually set back to node local
+ schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
+ schedulerApp.resetSchedulingOpportunities(prio, clock.getTime());
+ assertEquals(NodeType.NODE_LOCAL,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+
+ // Now escalate again to rack-local, then to off-switch
+ clock.tick(6);
+ assertEquals(NodeType.RACK_LOCAL,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+
+ clock.tick(7);
+ assertEquals(NodeType.OFF_SWITCH,
+ schedulerApp.getAllowedLocalityLevelByTime(prio,
+ nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
+ }
+
+ @Test
/**
* Ensure that when negative paramaters are given (signaling delay scheduling
* no tin use), the least restrictive locality level is returned.
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Wed Oct 30 22:21:59 2013
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.io.File;
@@ -41,7 +42,6 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -72,12 +73,14 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
@@ -286,6 +289,14 @@ public class TestFairScheduler {
conf.setBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, true);
conf.setFloat(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, .5f);
conf.setFloat(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7f);
+ conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED,
+ true);
+ conf.setInt(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS,
+ 10);
+ conf.setInt(FairSchedulerConfiguration.LOCALITY_DELAY_RACK_MS,
+ 5000);
+ conf.setInt(FairSchedulerConfiguration.LOCALITY_DELAY_NODE_MS,
+ 5000);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
@@ -296,6 +307,11 @@ public class TestFairScheduler {
Assert.assertEquals(true, scheduler.sizeBasedWeight);
Assert.assertEquals(.5, scheduler.nodeLocalityThreshold, .01);
Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01);
+ Assert.assertTrue("The continuous scheduling should be enabled",
+ scheduler.continuousSchedulingEnabled);
+ Assert.assertEquals(10, scheduler.continuousSchedulingSleepMs);
+ Assert.assertEquals(5000, scheduler.nodeLocalityDelayMs);
+ Assert.assertEquals(5000, scheduler.rackLocalityDelayMs);
Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory());
Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory());
Assert.assertEquals(128,
@@ -404,9 +420,9 @@ public class TestFairScheduler {
Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
assertEquals(3, queues.size());
- FSLeafQueue queue1 = queueManager.getLeafQueue("default");
- FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2");
- FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3");
+ FSLeafQueue queue1 = queueManager.getLeafQueue("default", true);
+ FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2", true);
+ FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true);
assertEquals(capacity / 2, queue1.getFairShare().getMemory());
assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB());
assertEquals(capacity / 4, queue2.getFairShare().getMemory());
@@ -418,25 +434,63 @@ public class TestFairScheduler {
@Test
public void testHierarchicalQueuesSimilarParents() {
QueueManager queueManager = scheduler.getQueueManager();
- FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child");
+ FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
Assert.assertNotNull(leafQueue);
Assert.assertEquals("root.parent.child", leafQueue.getName());
- FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent");
+ FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true);
Assert.assertNull(leafQueue2);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
- FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild");
+ FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true);
Assert.assertNull(leafQueue3);
Assert.assertEquals(2, queueManager.getLeafQueues().size());
- FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister");
+ FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true);
Assert.assertNotNull(leafQueue4);
Assert.assertEquals("root.parent.sister", leafQueue4.getName());
Assert.assertEquals(3, queueManager.getLeafQueues().size());
}
+ @Test
+ public void testSchedulerRootQueueMetrics() throws InterruptedException {
+
+ // Add a node
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue 1 requests full capacity of node
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ // Now queue 2 requests likewise
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure reserved memory gets updated correctly
+ assertEquals(1024, scheduler.rootMetrics.getReservedMB());
+
+ // Now another node checks in with capacity
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+ scheduler.handle(updateEvent2);
+
+
+ // The old reservation should still be there...
+ assertEquals(1024, scheduler.rootMetrics.getReservedMB());
+
+ // ... but it should disappear when we update the first node.
+ scheduler.handle(updateEvent);
+ assertEquals(0, scheduler.rootMetrics.getReservedMB());
+ }
+
@Test (timeout = 5000)
public void testSimpleContainerAllocation() {
// Add a node
@@ -539,24 +593,33 @@ public class TestFairScheduler {
Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
scheduler.reinitialize(conf, resourceManager.getRMContext());
+ RMContext rmContext = resourceManager.getRMContext();
+ Map<ApplicationId, RMApp> appsMap = rmContext.getRMApps();
+ ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
+ RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
+ null, null, null, ApplicationSubmissionContext.newInstance(null, null,
+ null, null, null, false, false, 0, null, null), null, null, 0, null);
+ appsMap.put(appAttemptId.getApplicationId(), rmApp);
+
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
- createAppAttemptId(1, 1), "default", "user1");
+ appAttemptId, "default", "user1");
scheduler.handle(appAddedEvent);
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("default")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
.getAppSchedulables().size());
+ assertEquals("root.user1", rmApp.getQueue());
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
scheduler.reinitialize(conf, resourceManager.getRMContext());
AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
createAppAttemptId(2, 1), "default", "user2");
scheduler.handle(appAddedEvent2);
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("default")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
.getAppSchedulables().size());
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true)
.getAppSchedulables().size());
}
@@ -702,7 +765,7 @@ public class TestFairScheduler {
assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
// That queue should have one app
- assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
@@ -712,7 +775,7 @@ public class TestFairScheduler {
scheduler.handle(appRemovedEvent1);
// Queue should have no apps
- assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1")
+ assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
.getAppSchedulables().size());
}
@@ -733,6 +796,7 @@ public class TestFairScheduler {
out.println("<queue name=\"queueB\">");
out.println("<minResources>2048mb,0vcores</minResources>");
out.println("<aclAdministerApps>alice,bob admins</aclAdministerApps>");
+ out.println("<schedulingPolicy>fair</schedulingPolicy>");
out.println("</queue>");
// Give queue C no minimum
out.println("<queue name=\"queueC\">");
@@ -759,6 +823,8 @@ public class TestFairScheduler {
+ "</defaultMinSharePreemptionTimeout>");
// Set fair share preemption timeout to 5 minutes
out.println("<fairSharePreemptionTimeout>300</fairSharePreemptionTimeout>");
+ // Set default scheduling policy to DRF
+ out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
out.println("</allocations>");
out.close();
@@ -791,22 +857,25 @@ public class TestFairScheduler {
assertEquals(10, queueManager.getUserMaxApps("user1"));
assertEquals(5, queueManager.getUserMaxApps("user2"));
+ // Root should get * ACL
+ assertEquals("*",queueManager.getQueueAcl("root",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals("*", queueManager.getQueueAcl("root",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
+
// Unspecified queues should get default ACL
- Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
- assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
- assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals(" ",queueManager.getQueueAcl("root.queueA",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
- assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-
- // Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
- assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueB",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+
+ // Queue C ACL
+ assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueC",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -817,6 +886,18 @@ public class TestFairScheduler {
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
+
+ // Verify existing queues have default scheduling policy
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getQueue("root").getPolicy().getName());
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getQueue("root.queueA").getPolicy().getName());
+ // Verify default is overriden if specified explicitly
+ assertEquals(FairSharePolicy.NAME,
+ queueManager.getQueue("root.queueB").getPolicy().getName());
+ // Verify new queue gets default scheduling policy
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getLeafQueue("root.newqueue", true).getPolicy().getName());
}
@Test
@@ -849,15 +930,71 @@ public class TestFairScheduler {
Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues();
Assert.assertEquals(4, leafQueues.size());
- Assert.assertNotNull(queueManager.getLeafQueue("queueA"));
- Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC"));
- Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD"));
- Assert.assertNotNull(queueManager.getLeafQueue("default"));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueA", false));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", false));
+ Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", false));
+ Assert.assertNotNull(queueManager.getLeafQueue("default", false));
// Make sure querying for queues didn't create any new ones:
Assert.assertEquals(4, leafQueues.size());
}
@Test
+ public void testConfigureRootQueue() throws Exception {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
+ out.println("<queue name=\"root\">");
+ out.println(" <schedulingPolicy>drf</schedulingPolicy>");
+ out.println(" <queue name=\"child1\">");
+ out.println(" <minResources>1024mb,1vcores</minResources>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"child2\">");
+ out.println(" <minResources>1024mb,4vcores</minResources>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ FSQueue root = queueManager.getRootQueue();
+ assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
+
+ assertNotNull(queueManager.getLeafQueue("child1", false));
+ assertNotNull(queueManager.getLeafQueue("child2", false));
+ }
+
+ /**
+ * Verify that you can't place queues at the same level as the root queue in
+ * the allocations file.
+ */
+ @Test (expected = AllocationConfigurationException.class)
+ public void testQueueAlongsideRoot() throws Exception {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\">");
+ out.println("</queue>");
+ out.println("<queue name=\"other\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+ }
+
+ @Test
public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
@@ -933,21 +1070,19 @@ public class TestFairScheduler {
assertEquals(5, queueManager.getUserMaxApps("user2"));
// Unspecified queues should get default ACL
- Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
- assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
- assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
- assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-
- // Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
- assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueB",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+
+ // Queue C ACL
+ assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueC",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
+
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -1353,9 +1488,9 @@ public class TestFairScheduler {
scheduler.update();
FSLeafQueue schedC =
- scheduler.getQueueManager().getLeafQueue("queueC");
+ scheduler.getQueueManager().getLeafQueue("queueC", true);
FSLeafQueue schedD =
- scheduler.getQueueManager().getLeafQueue("queueD");
+ scheduler.getQueueManager().getLeafQueue("queueD", true);
assertTrue(Resources.equals(
Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));
@@ -1534,8 +1669,13 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
+ out.println("<queue name=\"root\">");
+ out.println(" <aclSubmitApps> </aclSubmitApps>");
+ out.println(" <aclAdministerApps> </aclAdministerApps>");
+ out.println(" <queue name=\"queue1\">");
+ out.println(" <aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
+ out.println(" <aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
+ out.println(" </queue>");
out.println("</queue>");
out.println("</allocations>");
out.close();
@@ -1617,7 +1757,7 @@ public class TestFairScheduler {
FSSchedulerApp app1 = scheduler.applications.get(attId1);
FSSchedulerApp app2 = scheduler.applications.get(attId2);
- FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1");
+ FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true);
queue1.setPolicy(new FifoPolicy());
scheduler.update();
@@ -1645,7 +1785,7 @@ public class TestFairScheduler {
public void testMaxAssign() throws AllocationConfigurationException {
// set required scheduler configs
scheduler.assignMultiple = true;
- scheduler.getQueueManager().getLeafQueue("root.default")
+ scheduler.getQueueManager().getLeafQueue("root.default", true)
.setPolicy(SchedulingPolicy.getDefault());
RMNode node =
@@ -1722,7 +1862,7 @@ public class TestFairScheduler {
FSSchedulerApp app3 = scheduler.applications.get(attId3);
FSSchedulerApp app4 = scheduler.applications.get(attId4);
- scheduler.getQueueManager().getLeafQueue(fifoQueue)
+ scheduler.getQueueManager().getLeafQueue(fifoQueue, true)
.setPolicy(SchedulingPolicy.parse("fifo"));
scheduler.update();
@@ -1762,8 +1902,13 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<aclSubmitApps>userallow</aclSubmitApps>");
+ out.println("<queue name=\"root\">");
+ out.println(" <aclSubmitApps> </aclSubmitApps>");
+ out.println(" <aclAdministerApps> </aclAdministerApps>");
+ out.println(" <queue name=\"queue1\">");
+ out.println(" <aclSubmitApps>userallow</aclSubmitApps>");
+ out.println(" <aclAdministerApps>userallow</aclAdministerApps>");
+ out.println(" </queue>");
out.println("</queue>");
out.println("</allocations>");
out.close();
@@ -2196,4 +2341,149 @@ public class TestFairScheduler {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
+ public void testConcurrentAccessOnApplications() throws Exception {
+ FairScheduler fs = new FairScheduler();
+ TestCapacityScheduler.verifyConcurrentAccessOnApplications(
+ fs.applications, FSSchedulerApp.class);
+ }
+
+ @Test (timeout = 5000)
+ public void testContinuousScheduling() throws Exception {
+ // set continuous scheduling enabled
+ FairScheduler fs = new FairScheduler();
+ Configuration conf = createConfiguration();
+ conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED,
+ true);
+ fs.reinitialize(conf, resourceManager.getRMContext());
+ Assert.assertTrue("Continuous scheduling should be enabled.",
+ fs.isContinuousSchedulingEnabled());
+
+ // Add one node
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ fs.handle(nodeEvent1);
+
+ // available resource
+ Assert.assertEquals(fs.getClusterCapacity().getMemory(), 8 * 1024);
+ Assert.assertEquals(fs.getClusterCapacity().getVirtualCores(), 8);
+
+ // send application request
+ ApplicationAttemptId appAttemptId =
+ createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
+ fs.addApplication(appAttemptId, "queue11", "user11");
+ List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+ ResourceRequest request =
+ createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
+ ask.add(request);
+ fs.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
+
+ // waiting for continuous_scheduler_sleep_time
+ // at least one pass
+ Thread.sleep(fs.getConf().getContinuousSchedulingSleepMs() + 500);
+
+ FSSchedulerApp app = fs.applications.get(appAttemptId);
+ // Wait until app gets resources.
+ while (app.getCurrentConsumption().equals(Resources.none())) { }
+
+ // check consumption
+ Assert.assertEquals(1024, app.getCurrentConsumption().getMemory());
+ Assert.assertEquals(1, app.getCurrentConsumption().getVirtualCores());
+ }
+
+
+ @Test
+ public void testDontAllowUndeclaredPools() throws Exception{
+ Configuration conf = createConfiguration();
+ conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"jerry\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false);
+ FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false);
+
+ // Should get put into jerry
+ createSchedulingRequest(1024, "jerry", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+
+ // Should get forced into default
+ createSchedulingRequest(1024, "newqueue", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+ assertEquals(1, defaultQueue.getAppSchedulables().size());
+
+ // Would get put into someuser because of user-as-default-queue, but should
+ // be forced into default
+ createSchedulingRequest(1024, "default", "someuser");
+ assertEquals(1, jerryQueue.getAppSchedulables().size());
+ assertEquals(2, defaultQueue.getAppSchedulables().size());
+
+ // Should get put into jerry because of user-as-default-queue
+ createSchedulingRequest(1024, "default", "jerry");
+ assertEquals(2, jerryQueue.getAppSchedulables().size());
+ assertEquals(2, defaultQueue.getAppSchedulables().size());
+ }
+
+ @SuppressWarnings("resource")
+ @Test
+ public void testBlacklistNodes() throws Exception {
+ final int GB = 1024;
+ String host = "127.0.0.1";
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(16 * GB, 16),
+ 0, host);
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+
+ ApplicationAttemptId appAttemptId =
+ createSchedulingRequest(GB, "root.default", "user", 1);
+ FSSchedulerApp app = scheduler.applications.get(appAttemptId);
+
+ // Verify the blacklist can be updated independent of requesting containers
+ scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
+ Collections.<ContainerId>emptyList(),
+ Collections.singletonList(host), null);
+ assertTrue(app.isBlacklisted(host));
+ scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
+ Collections.<ContainerId>emptyList(), null,
+ Collections.singletonList(host));
+ assertFalse(scheduler.applications.get(appAttemptId).isBlacklisted(host));
+
+ List<ResourceRequest> update = Arrays.asList(
+ createResourceRequest(GB, node.getHostName(), 1, 0, true));
+
+ // Verify a container does not actually get placed on the blacklisted host
+ scheduler.allocate(appAttemptId, update,
+ Collections.<ContainerId>emptyList(),
+ Collections.singletonList(host), null);
+ assertTrue(app.isBlacklisted(host));
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 0, app
+ .getLiveContainers().size());
+
+ // Verify a container gets placed on the empty blacklist
+ scheduler.allocate(appAttemptId, update,
+ Collections.<ContainerId>emptyList(), null,
+ Collections.singletonList(host));
+ assertFalse(app.isBlacklisted(host));
+ createSchedulingRequest(GB, "root.default", "user", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 1, app
+ .getLiveContainers().size());
+ }
}