You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by at...@apache.org on 2013/04/13 01:05:38 UTC
svn commit: r1467511 [5/6] - in
/hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Fri Apr 12 23:05:28 2013
@@ -77,6 +77,7 @@ public class AMLauncher implements Runna
RecordFactoryProvider.getRecordFactory(null);
private final AMLauncherEventType eventType;
private final RMContext rmContext;
+ private final Container masterContainer;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -88,34 +89,36 @@ public class AMLauncher implements Runna
this.eventType = eventType;
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
+ this.masterContainer = application.getMasterContainer();
}
private void connect() throws IOException {
- ContainerId masterContainerID = application.getMasterContainer().getId();
+ ContainerId masterContainerID = masterContainer.getId();
containerMgrProxy = getContainerMgrProxy(masterContainerID);
}
private void launch() throws IOException {
connect();
- ContainerId masterContainerID = application.getMasterContainer().getId();
+ ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext =
application.getSubmissionContext();
- LOG.info("Setting up container " + application.getMasterContainer()
+ LOG.info("Setting up container " + masterContainer
+ " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
createAMContainerLaunchContext(applicationContext, masterContainerID);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
+ request.setContainer(masterContainer);
containerMgrProxy.startContainer(request);
- LOG.info("Done launching container " + application.getMasterContainer()
+ LOG.info("Done launching container " + masterContainer
+ " for AM " + application.getAppAttemptId());
}
private void cleanup() throws IOException {
connect();
- ContainerId containerId = application.getMasterContainer().getId();
+ ContainerId containerId = masterContainer.getId();
StopContainerRequest stopRequest =
recordFactory.newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(containerId);
@@ -126,9 +129,7 @@ public class AMLauncher implements Runna
protected ContainerManager getContainerMgrProxy(
final ContainerId containerId) {
- Container container = application.getMasterContainer();
-
- final NodeId node = container.getNodeId();
+ final NodeId node = masterContainer.getNodeId();
final InetSocketAddress containerManagerBindAddress =
NetUtils.createSocketAddrForHost(node.getHost(), node.getPort());
@@ -138,8 +139,8 @@ public class AMLauncher implements Runna
.createRemoteUser(containerId.toString());
if (UserGroupInformation.isSecurityEnabled()) {
Token<ContainerTokenIdentifier> token =
- ProtoUtils.convertFromProtoFormat(container.getContainerToken(),
- containerManagerBindAddress);
+ ProtoUtils.convertFromProtoFormat(masterContainer
+ .getContainerToken(), containerManagerBindAddress);
currentUser.addToken(token);
}
return currentUser.doAs(new PrivilegedAction<ContainerManager>() {
@@ -165,30 +166,28 @@ public class AMLauncher implements Runna
new String[0])));
// Finalize the container
- container.setContainerId(containerID);
- container.setUser(applicationMasterContext.getUser());
- setupTokensAndEnv(container);
+ container.setUser(applicationMasterContext.getAMContainerSpec().getUser());
+ setupTokensAndEnv(container, containerID);
return container;
}
private void setupTokensAndEnv(
- ContainerLaunchContext container)
+ ContainerLaunchContext container, ContainerId containerID)
throws IOException {
Map<String, String> environment = container.getEnvironment();
-
environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
application.getWebProxyBase());
// Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be
// consumable by the AM.
- environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, container
- .getContainerId().toString());
- environment.put(ApplicationConstants.NM_HOST_ENV, application
- .getMasterContainer().getNodeId().getHost());
+ environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV,
+ containerID.toString());
+ environment.put(ApplicationConstants.NM_HOST_ENV, masterContainer
+ .getNodeId().getHost());
environment.put(ApplicationConstants.NM_PORT_ENV,
- String.valueOf(application.getMasterContainer().getNodeId().getPort()));
+ String.valueOf(masterContainer.getNodeId().getPort()));
String parts[] =
- application.getMasterContainer().getNodeHttpAddress().split(":");
+ masterContainer.getNodeHttpAddress().split(":");
environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]);
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java Fri Apr 12 23:05:28 2013
@@ -53,12 +53,14 @@ public class DefaultResourceCalculator e
}
@Override
- public Resource normalize(Resource r, Resource minimumResource) {
- return Resources.createResource(
+ public Resource normalize(Resource r, Resource minimumResource,
+ Resource maximumResource) {
+ int normalizedMemory = Math.min(
roundUp(
Math.max(r.getMemory(), minimumResource.getMemory()),
- minimumResource.getMemory())
- );
+ minimumResource.getMemory()),
+ maximumResource.getMemory());
+ return Resources.createResource(normalizedMemory);
}
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java Fri Apr 12 23:05:28 2013
@@ -123,15 +123,20 @@ public class DominantResourceCalculator
}
@Override
- public Resource normalize(Resource r, Resource minimumResource) {
- return Resources.createResource(
+ public Resource normalize(Resource r, Resource minimumResource,
+ Resource maximumResource) {
+ int normalizedMemory = Math.min(
roundUp(
- Math.max(r.getMemory(), minimumResource.getMemory()),
+ Math.max(r.getMemory(), minimumResource.getMemory()),
minimumResource.getMemory()),
+ maximumResource.getMemory());
+ int normalizedCores = Math.min(
roundUp(
Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()),
- minimumResource.getVirtualCores())
- );
+ minimumResource.getVirtualCores()),
+ maximumResource.getVirtualCores());
+ return Resources.createResource(normalizedMemory,
+ normalizedCores);
}
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java Fri Apr 12 23:05:28 2013
@@ -88,13 +88,16 @@ public abstract class ResourceCalculator
/**
* Normalize resource <code>r</code> given the base
- * <code>minimumResource</code>.
+ * <code>minimumResource</code> and verify against max allowed
+ * <code>maximumResource</code>
*
* @param r resource
* @param minimumResource step-factor
+ * @param maximumResource the upper bound of the resource to be allocated
* @return normalized resource
*/
- public abstract Resource normalize(Resource r, Resource minimumResource);
+ public abstract Resource normalize(Resource r, Resource minimumResource,
+ Resource maximumResource);
/**
* Round-up resource <code>r</code> given factor <code>stepFactor</code>.
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Fri Apr 12 23:05:28 2013
@@ -132,8 +132,9 @@ public class Resources {
}
public static Resource normalize(
- ResourceCalculator calculator, Resource lhs, Resource factor) {
- return calculator.normalize(lhs, factor);
+ ResourceCalculator calculator, Resource lhs, Resource factor,
+ Resource limit) {
+ return calculator.normalize(lhs, factor, limit);
}
public static Resource roundUp(
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Apr 12 23:05:28 2013
@@ -690,7 +690,7 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(
new AppAddedSchedulerEvent(appAttempt.applicationAttemptId,
appAttempt.submissionContext.getQueue(),
- appAttempt.submissionContext.getUser()));
+ appAttempt.submissionContext.getAMContainerSpec().getUser()));
}
}
@@ -736,10 +736,13 @@ public class RMAppAttemptImpl implements
RMAppEventType.APP_ACCEPTED));
// Request a container for the AM.
- ResourceRequest request = BuilderUtils.newResourceRequest(
- AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt.submissionContext
- .getAMContainerSpec().getResource(), 1);
+ ResourceRequest request =
+ BuilderUtils.newResourceRequest(
+ AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt
+ .getSubmissionContext().getResource(), 1);
+ // SchedulerUtils.validateResourceRequests is not necessary because
+ // AM resource has been checked when submission
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId,
Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST);
@@ -771,7 +774,8 @@ public class RMAppAttemptImpl implements
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
-
+ appAttempt.getSubmissionContext().setResource(
+ appAttempt.getMasterContainer().getResource());
RMStateStore store = appAttempt.rmContext.getStateStore();
appAttempt.storeAttempt(store);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Fri Apr 12 23:05:28 2013
@@ -288,14 +288,9 @@ public class RMContainerImpl implements
public void transition(RMContainerImpl container, RMContainerEvent event) {
RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
- // Update container-status for diagnostics. Today we completely
- // replace it on finish. We may just need to update diagnostics.
- container.container.setContainerStatus(finishedEvent
- .getRemoteContainerStatus());
-
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
- container.appAttemptId, container.container.getContainerStatus()));
+ container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Fri Apr 12 23:05:28 2013
@@ -89,10 +89,12 @@ public class SchedulerUtils {
List<ResourceRequest> asks,
ResourceCalculator resourceCalculator,
Resource clusterResource,
- Resource minimumResource) {
+ Resource minimumResource,
+ Resource maximumResource) {
for (ResourceRequest ask : asks) {
normalizeRequest(
- ask, resourceCalculator, clusterResource, minimumResource);
+ ask, resourceCalculator, clusterResource, minimumResource,
+ maximumResource);
}
}
@@ -104,12 +106,50 @@ public class SchedulerUtils {
ResourceRequest ask,
ResourceCalculator resourceCalculator,
Resource clusterResource,
- Resource minimumResource) {
+ Resource minimumResource,
+ Resource maximumResource) {
Resource normalized =
Resources.normalize(
- resourceCalculator, ask.getCapability(), minimumResource);
- ask.getCapability().setMemory(normalized.getMemory());
- ask.getCapability().setVirtualCores(normalized.getVirtualCores());
+ resourceCalculator, ask.getCapability(), minimumResource,
+ maximumResource);
+ ask.setCapability(normalized);
+ }
+
+ /**
+ * Utility method to validate a resource request, by insuring that the
+ * requested memory/vcore is non-negative and not greater than max
+ */
+ public static void validateResourceRequest(ResourceRequest resReq,
+ Resource maximumResource) throws InvalidResourceRequestException {
+ if (resReq.getCapability().getMemory() < 0 ||
+ resReq.getCapability().getMemory() > maximumResource.getMemory()) {
+ throw new InvalidResourceRequestException("Invalid resource request"
+ + ", requested memory < 0"
+ + ", or requested memory > max configured"
+ + ", requestedMemory=" + resReq.getCapability().getMemory()
+ + ", maxMemory=" + maximumResource.getMemory());
+ }
+ if (resReq.getCapability().getVirtualCores() < 0 ||
+ resReq.getCapability().getVirtualCores() >
+ maximumResource.getVirtualCores()) {
+ throw new InvalidResourceRequestException("Invalid resource request"
+ + ", requested virtual cores < 0"
+ + ", or requested virtual cores > max configured"
+ + ", requestedVirtualCores="
+ + resReq.getCapability().getVirtualCores()
+ + ", maxVirtualCores=" + maximumResource.getVirtualCores());
+ }
+ }
+
+ /**
+ * Utility method to validate a list resource requests, by insuring that the
+ * requested memory/vcore is non-negative and not greater than max
+ */
+ public static void validateResourceRequests(List<ResourceRequest> ask,
+ Resource maximumResource) throws InvalidResourceRequestException {
+ for (ResourceRequest resReq : ask) {
+ validateResourceRequest(resReq, maximumResource);
+ }
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Apr 12 23:05:28 2013
@@ -108,7 +108,7 @@ implements ResourceScheduler, CapacitySc
new Comparator<FiCaSchedulerApp>() {
@Override
public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
- return a1.getApplicationId().getId() - a2.getApplicationId().getId();
+ return a1.getApplicationId().compareTo(a2.getApplicationId());
}
};
@@ -483,7 +483,8 @@ implements ResourceScheduler, CapacitySc
// Sanity check
SchedulerUtils.normalizeRequests(
- ask, calculator, getClusterResources(), minimumAllocation);
+ ask, calculator, getClusterResources(), minimumAllocation,
+ maximumAllocation);
// Release containers
for (ContainerId releasedContainerId : release) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Fri Apr 12 23:05:28 2013
@@ -315,8 +315,8 @@ public class CapacitySchedulerConfigurat
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
int maximumCores = getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES);
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
return Resources.createResource(maximumMemory, maximumCores);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Apr 12 23:05:28 2013
@@ -89,6 +89,8 @@ public class LeafQueue implements CSQueu
private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
+ private int nodeLocalityDelay;
+
private Resource usedResources = Resources.createResource(0, 0);
private float usedCapacity = 0.0f;
private volatile int numContainers;
@@ -123,8 +125,6 @@ public class LeafQueue implements CSQueu
private final ActiveUsersManager activeUsersManager;
- private final int nodeLocalityDelay;
-
private final ResourceCalculator resourceCalculator;
public LeafQueue(CapacitySchedulerContext cs,
@@ -196,9 +196,6 @@ public class LeafQueue implements CSQueu
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
- this.nodeLocalityDelay =
- cs.getConfiguration().getNodeLocalityDelay();
-
setupQueueConfigs(
cs.getClusterResources(),
capacity, absoluteCapacity,
@@ -206,7 +203,7 @@ public class LeafQueue implements CSQueu
userLimit, userLimitFactor,
maxApplications, maxApplicationsPerUser,
maxActiveApplications, maxActiveApplicationsPerUser,
- state, acls);
+ state, acls, cs.getConfiguration().getNodeLocalityDelay());
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
@@ -227,7 +224,8 @@ public class LeafQueue implements CSQueu
int userLimit, float userLimitFactor,
int maxApplications, int maxApplicationsPerUser,
int maxActiveApplications, int maxActiveApplicationsPerUser,
- QueueState state, Map<QueueACL, AccessControlList> acls)
+ QueueState state, Map<QueueACL, AccessControlList> acls,
+ int nodeLocalityDelay)
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -256,6 +254,8 @@ public class LeafQueue implements CSQueu
this.queueInfo.setCapacity(this.capacity);
this.queueInfo.setMaximumCapacity(this.maximumCapacity);
this.queueInfo.setQueueState(this.state);
+
+ this.nodeLocalityDelay = nodeLocalityDelay;
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -319,7 +319,8 @@ public class LeafQueue implements CSQueu
"state = " + state +
" [= configuredState ]" + "\n" +
"acls = " + aclsString +
- " [= configuredAcls ]" + "\n");
+ " [= configuredAcls ]" + "\n" +
+ "nodeLocalityDelay = " + nodeLocalityDelay + "\n");
}
@Override
@@ -605,7 +606,8 @@ public class LeafQueue implements CSQueu
newlyParsedLeafQueue.getMaxApplicationsPerUser(),
newlyParsedLeafQueue.getMaximumActiveApplications(),
newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
- newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
+ newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
+ newlyParsedLeafQueue.getNodeLocalityDelay());
// queue metrics are updated, more resource may be available
// activate the pending applications if possible
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Fri Apr 12 23:05:28 2013
@@ -650,11 +650,12 @@ public class FairScheduler implements Re
*
* @param asks a list of resource requests
* @param minMemory the configured minimum memory allocation
+ * @param maxMemory the configured maximum memory allocation
*/
static void normalizeRequests(List<ResourceRequest> asks,
- int minMemory) {
+ int minMemory, int maxMemory) {
for (ResourceRequest ask : asks) {
- normalizeRequest(ask, minMemory);
+ normalizeRequest(ask, minMemory, maxMemory);
}
}
@@ -664,11 +665,14 @@ public class FairScheduler implements Re
*
* @param ask the resource request
* @param minMemory the configured minimum memory allocation
+ * @param maxMemory the configured maximum memory allocation
*/
- static void normalizeRequest(ResourceRequest ask, int minMemory) {
+ static void normalizeRequest(ResourceRequest ask, int minMemory,
+ int maxMemory) {
int memory = Math.max(ask.getCapability().getMemory(), minMemory);
- ask.getCapability().setMemory(
- minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0)));
+ int normalizedMemory =
+ minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0));
+ ask.getCapability().setMemory(Math.min(normalizedMemory, maxMemory));
}
@Override
@@ -684,7 +688,8 @@ public class FairScheduler implements Re
}
// Sanity check
- normalizeRequests(ask, minimumAllocation.getMemory());
+ normalizeRequests(ask, minimumAllocation.getMemory(),
+ maximumAllocation.getMemory());
// Release containers
for (ContainerId releasedContainerId : release) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Apr 12 23:05:28 2013
@@ -232,7 +232,7 @@ public class FifoScheduler implements Re
// Sanity check
SchedulerUtils.normalizeRequests(ask, resourceCalculator,
- clusterResource, minimumAllocation);
+ clusterResource, minimumAllocation, maximumAllocation);
// Release containers
for (ContainerId releasedContainer : release) {
@@ -462,7 +462,7 @@ public class FifoScheduler implements Re
FiCaSchedulerApp application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
+ application.getResourceRequest(priority, node.getHostName());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest =
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java Fri Apr 12 23:05:28 2013
@@ -59,8 +59,8 @@ public class AppAttemptInfo {
this.logsLink = join(HttpConfig.getSchemePrefix(),
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
- ConverterUtils.toString(masterContainer.getId()),
- "/", attempt.getSubmissionContext().getUser());
+ ConverterUtils.toString(masterContainer.getId()), "/",
+ attempt.getSubmissionContext().getAMContainerSpec().getUser());
}
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Fri Apr 12 23:05:28 2013
@@ -130,7 +130,7 @@ public class Application {
public synchronized void submit() throws IOException {
ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
context.setApplicationId(this.applicationId);
- context.setUser(this.user);
+ context.getAMContainerSpec().setUser(this.user);
context.setQueue(this.queue);
SubmitApplicationRequest request = recordFactory
.newRecordInstance(SubmitApplicationRequest.class);
@@ -340,7 +340,8 @@ public class Application {
// Launch the container
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(createCLC(container));
+ startRequest.setContainerLaunchContext(createCLC());
+ startRequest.setContainer(container);
nodeManager.startContainer(startRequest);
break;
}
@@ -396,11 +397,9 @@ public class Application {
}
}
- private ContainerLaunchContext createCLC(Container container) {
+ private ContainerLaunchContext createCLC() {
ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
- clc.setContainerId(container.getId());
clc.setUser(this.user);
- clc.setResource(container.getResource());
return clc;
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Fri Apr 12 23:05:28 2013
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
@@ -71,11 +70,11 @@ public class MockNM {
this.resourceTracker = resourceTracker;
}
- public void containerStatus(Container container) throws Exception {
+ public void containerStatus(ContainerStatus containerStatus) throws Exception {
Map<ApplicationId, List<ContainerStatus>> conts =
new HashMap<ApplicationId, List<ContainerStatus>>();
- conts.put(container.getId().getApplicationAttemptId().getApplicationId(),
- Arrays.asList(new ContainerStatus[] { container.getContainerStatus() }));
+ conts.put(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(),
+ Arrays.asList(new ContainerStatus[] { containerStatus }));
nodeHeartbeat(conts, true);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Fri Apr 12 23:05:28 2013
@@ -209,6 +209,7 @@ public class MockNodes {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String hostName = "host"+ nid;
+ final String nodeAddr = hostName + ":" + nid;
final int port = 123;
final NodeId nodeID = newNodeID(hostName, port);
final String httpAddress = httpAddr;
@@ -218,7 +219,7 @@ public class MockNodes {
nodeHealthStatus.setIsNodeHealthy(true);
nodeHealthStatus.setHealthReport("HealthyMe");
}
- return new MockRMNodeImpl(nodeID, hostName, httpAddress, perNode, rackName,
+ return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, rackName,
nodeHealthStatus, nid, hostName, state);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Apr 12 23:05:28 2013
@@ -128,21 +128,28 @@ public class MockRM extends ResourceMana
// client
public RMApp submitApp(int masterMemory, String name, String user) throws Exception {
- return submitApp(masterMemory, name, user, null, false, null);
+ return submitApp(masterMemory, name, user, null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
}
public RMApp submitApp(int masterMemory, String name, String user,
Map<ApplicationAccessType, String> acls) throws Exception {
- return submitApp(masterMemory, name, user, acls, false, null);
+ return submitApp(masterMemory, name, user, acls, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
}
public RMApp submitApp(int masterMemory, String name, String user,
Map<ApplicationAccessType, String> acls, String queue) throws Exception {
- return submitApp(masterMemory, name, user, acls, false, queue);
- }
+ return submitApp(masterMemory, name, user, acls, false, queue,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
+ }
public RMApp submitApp(int masterMemory, String name, String user,
- Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue) throws Exception {
+ Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
+ int maxAppAttempts) throws Exception {
ClientRMProtocol client = getClientRMService();
GetNewApplicationResponse resp = client.getNewApplication(Records
.newRecord(GetNewApplicationRequest.class));
@@ -154,7 +161,7 @@ public class MockRM extends ResourceMana
.newRecord(ApplicationSubmissionContext.class);
sub.setApplicationId(appId);
sub.setApplicationName(name);
- sub.setUser(user);
+ sub.setMaxAppAttempts(maxAppAttempts);
if(unmanaged) {
sub.setUnmanagedAM(true);
}
@@ -163,13 +170,13 @@ public class MockRM extends ResourceMana
}
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
- Resource capability = Records.newRecord(Resource.class);
+ final Resource capability = Records.newRecord(Resource.class);
capability.setMemory(masterMemory);
- clc.setResource(capability);
+ sub.setResource(capability);
clc.setApplicationACLs(acls);
+ clc.setUser(user);
sub.setAMContainerSpec(clc);
req.setApplicationSubmissionContext(sub);
-
UserGroupInformation fakeUser =
UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
PrivilegedAction<SubmitApplicationResponse> action =
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Fri Apr 12 23:05:28 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
@@ -76,6 +75,9 @@ public class NodeManager implements Cont
final Map<ApplicationId, List<Container>> containers =
new HashMap<ApplicationId, List<Container>>();
+ final Map<Container, ContainerStatus> containerStatusMap =
+ new HashMap<Container, ContainerStatus>();
+
public NodeManager(String hostName, int containerManagerPort, int httpPort,
String rackName, Resource capability,
ResourceTrackerService resourceTrackerService, RMContext rmContext)
@@ -137,7 +139,7 @@ public class NodeManager implements Cont
List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
for (List<Container> appContainers : containers.values()) {
for (Container container : appContainers) {
- containerStatuses.add(container.getContainerStatus());
+ containerStatuses.add(containerStatusMap.get(container));
}
}
return containerStatuses;
@@ -159,11 +161,10 @@ public class NodeManager implements Cont
synchronized public StartContainerResponse startContainer(
StartContainerRequest request)
throws YarnRemoteException {
- ContainerLaunchContext containerLaunchContext =
- request.getContainerLaunchContext();
-
+ Container requestContainer = request.getContainer();
+
ApplicationId applicationId =
- containerLaunchContext.getContainerId().getApplicationAttemptId().
+ requestContainer.getId().getApplicationAttemptId().
getApplicationId();
List<Container> applicationContainers = containers.get(applicationId);
@@ -174,25 +175,28 @@ public class NodeManager implements Cont
// Sanity check
for (Container container : applicationContainers) {
- if (container.getId().compareTo(containerLaunchContext.getContainerId())
+ if (container.getId().compareTo(requestContainer.getId())
== 0) {
throw new IllegalStateException(
- "Container " + containerLaunchContext.getContainerId() +
+ "Container " + requestContainer.getId() +
" already setup on node " + containerManagerAddress);
}
}
Container container =
- BuilderUtils.newContainer(containerLaunchContext.getContainerId(),
+ BuilderUtils.newContainer(requestContainer.getId(),
this.nodeId, nodeHttpAddress,
- containerLaunchContext.getResource(),
+ requestContainer.getResource(),
null, null // DKDC - Doesn't matter
);
+ ContainerStatus containerStatus =
+ BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW,
+ "", -1000);
applicationContainers.add(container);
-
- Resources.subtractFrom(available, containerLaunchContext.getResource());
- Resources.addTo(used, containerLaunchContext.getResource());
+ containerStatusMap.put(container, containerStatus);
+ Resources.subtractFrom(available, requestContainer.getResource());
+ Resources.addTo(used, requestContainer.getResource());
if(LOG.isDebugEnabled()) {
LOG.debug("startContainer:" + " node=" + containerManagerAddress
@@ -223,7 +227,9 @@ public class NodeManager implements Cont
List<Container> applicationContainers = containers.get(applicationId);
for (Container c : applicationContainers) {
if (c.getId().compareTo(containerID) == 0) {
- c.setState(ContainerState.COMPLETE);
+ ContainerStatus containerStatus = containerStatusMap.get(c);
+ containerStatus.setState(ContainerState.COMPLETE);
+ containerStatusMap.put(c, containerStatus);
}
}
@@ -277,8 +283,8 @@ public class NodeManager implements Cont
}
GetContainerStatusResponse response =
recordFactory.newRecordInstance(GetContainerStatusResponse.class);
- if (container != null && container.getContainerStatus() != null) {
- response.setStatus(container.getContainerStatus());
+ if (container != null && containerStatusMap.get(container).getState() != null) {
+ response.setStatus(containerStatusMap.get(container));
}
return response;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Fri Apr 12 23:05:28 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.Service;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -503,6 +504,10 @@ public class TestAppManager{
RMApp appOrig = rmContext.getRMApps().get(appID);
Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());
+ ContainerLaunchContext clc =
+ BuilderUtils.newContainerLaunchContext(null, null, null, null, null,
+ null, null);
+ context.setAMContainerSpec(clc);
// our testApp1 should be rejected and original app with same id should be left in place
appMonitor.submitApplication(context);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Fri Apr 12 23:05:28 2013
@@ -169,7 +169,7 @@ public class TestApplicationACLs {
ContainerLaunchContext amContainer = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
Resource resource = BuilderUtils.newResource(1024, 1);
- amContainer.setResource(resource);
+ context.setResource(resource);
amContainer.setApplicationACLs(acls);
context.setAMContainerSpec(amContainer);
submitRequest.setApplicationSubmissionContext(context);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Fri Apr 12 23:05:28 2013
@@ -134,7 +134,7 @@ public class TestApplicationMasterLaunch
Assert.assertEquals(app.getSubmitTime(),
containerManager.submitTimeAtContainerManager);
Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
- .getSubmissionContext().getAMContainerSpec().getContainerId()
+ .getMasterContainer().getId()
.toString(), containerManager.containerIdAtContainerManager);
Assert.assertEquals(nm1.getNodeId().getHost(),
containerManager.nmHostAtContainerManager);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Fri Apr 12 23:05:28 2013
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.ipc.YarnRP
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
@@ -254,6 +255,12 @@ public class TestClientRMService {
public void testConcurrentAppSubmit()
throws IOException, InterruptedException, BrokenBarrierException {
YarnScheduler yarnScheduler = mock(YarnScheduler.class);
+ when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
+ when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
RMContext rmContext = mock(RMContext.class);
mockRMContext(yarnScheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
@@ -311,21 +318,63 @@ public class TestClientRMService {
endBarrier.await();
t.join();
}
+
+ @Test (timeout = 30000)
+ public void testInvalidResourceRequestWhenSubmittingApplication()
+ throws IOException, InterruptedException, BrokenBarrierException {
+ YarnScheduler yarnScheduler = mock(YarnScheduler.class);
+ when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
+ when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+ RMContext rmContext = mock(RMContext.class);
+ mockRMContext(yarnScheduler, rmContext);
+ RMStateStore stateStore = mock(RMStateStore.class);
+ when(rmContext.getStateStore()).thenReturn(stateStore);
+ RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
+ null, mock(ApplicationACLsManager.class), new Configuration());
+
+ final ApplicationId appId = getApplicationId(100);
+ final SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId);
+ Resource resource = Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1);
+ when(submitRequest.getApplicationSubmissionContext()
+ .getResource()).thenReturn(resource);
+
+ final ClientRMService rmService =
+ new ClientRMService(rmContext, yarnScheduler, appManager, null, null);
+
+ // submit an app
+ try {
+ rmService.submitApplication(submitRequest);
+ Assert.fail("Application submission should fail because resource" +
+ " request is invalid.");
+ } catch (YarnRemoteException e) {
+ // Exception is expected
+ Assert.assertTrue("The thrown exception is not" +
+ " InvalidResourceRequestException",
+ e.getMessage().startsWith("Invalid resource request"));
+ }
+ }
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId) {
String user = MockApps.newUserName();
String queue = MockApps.newQueue();
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
- Resource resource = mock(Resource.class);
- when(amContainerSpec.getResource()).thenReturn(resource);
+
+ Resource resource = Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
- when(submissionContext.getUser()).thenReturn(user);
- when(submissionContext.getQueue()).thenReturn(queue);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
+ when(submissionContext.getAMContainerSpec().getUser()).thenReturn(user);
+ when(submissionContext.getQueue()).thenReturn(queue);
when(submissionContext.getApplicationId()).thenReturn(appId);
-
+ when(submissionContext.getResource()).thenReturn(resource);
+
SubmitApplicationRequest submitRequest =
recordFactory.newRecordInstance(SubmitApplicationRequest.class);
submitRequest.setApplicationSubmissionContext(submissionContext);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Fri Apr 12 23:05:28 2013
@@ -139,8 +139,9 @@ public class TestFifoScheduler {
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemory());
- c1.setState(ContainerState.COMPLETE);
- nm1.containerStatus(c1);
+ ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
+ c1.getId(), ContainerState.COMPLETE, "", 0);
+ nm1.containerStatus(containerStatus);
int waitCount = 0;
while (attempt1.getJustFinishedContainers().size() < 1
&& waitCount++ != 20) {
@@ -197,6 +198,8 @@ public class TestFifoScheduler {
int allocMB = 1536;
YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ allocMB * 10);
// Test for something lesser than this.
testMinimumAllocation(conf, allocMB / 2);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Fri Apr 12 23:05:28 2013
@@ -19,11 +19,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -62,6 +64,7 @@ public class TestRMRestart {
"org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore");
conf.set(YarnConfiguration.RM_SCHEDULER,
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 5);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
@@ -152,7 +155,9 @@ public class TestRMRestart {
.getApplicationId());
// create unmanaged app
- RMApp appUnmanaged = rm1.submitApp(200, "someApp", "someUser", null, true, null);
+ RMApp appUnmanaged = rm1.submitApp(200, "someApp", "someUser", null, true,
+ null, conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS));
ApplicationAttemptId unmanagedAttemptId =
appUnmanaged.getCurrentAppAttempt().getAppAttemptId();
// assert appUnmanaged info is saved
@@ -220,9 +225,9 @@ public class TestRMRestart {
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true);
- Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
+ Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
hbResponse = nm2.nodeHeartbeat(true);
- Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction());
+ Assert.assertEquals(NodeAction.RESYNC, hbResponse.getNodeAction());
// new NM to represent NM re-register
nm1 = rm2.registerNode("h1:1234", 15120);
@@ -230,9 +235,9 @@ public class TestRMRestart {
// verify no more reboot response sent
hbResponse = nm1.nodeHeartbeat(true);
- Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
+ Assert.assertTrue(NodeAction.RESYNC != hbResponse.getNodeAction());
hbResponse = nm2.nodeHeartbeat(true);
- Assert.assertTrue(NodeAction.REBOOT != hbResponse.getNodeAction());
+ Assert.assertTrue(NodeAction.RESYNC != hbResponse.getNodeAction());
// assert app1 attempt is saved
attempt1 = loadedApp1.getCurrentAppAttempt();
@@ -306,4 +311,74 @@ public class TestRMRestart {
Assert.assertEquals(0, rmAppState.size());
}
+ @Test
+ public void testRMRestartOnMaxAppAttempts() throws Exception {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ ExitUtil.disableSystemExit();
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
+ conf.set(YarnConfiguration.RM_STORE,
+ "org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore");
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ RMState rmState = memStore.getState();
+
+ Map<ApplicationId, ApplicationState> rmAppState =
+ rmState.getApplicationState();
+ MockRM rm1 = new MockRM(conf, memStore);
+ rm1.start();
+ MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // submit an app with maxAppAttempts equals to 1
+ RMApp app1 = rm1.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default", 1);
+ // submit an app with maxAppAttempts equals to -1
+ RMApp app2 = rm1.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), false, "default", -1);
+
+ // assert app1 info is saved
+ ApplicationState appState = rmAppState.get(app1.getApplicationId());
+ Assert.assertNotNull(appState);
+ Assert.assertEquals(0, appState.getAttemptCount());
+ Assert.assertEquals(appState.getApplicationSubmissionContext()
+ .getApplicationId(), app1.getApplicationSubmissionContext()
+ .getApplicationId());
+
+ // Allocate the AM
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt = app1.getCurrentAppAttempt();
+ ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
+ rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
+ Assert.assertEquals(1, appState.getAttemptCount());
+ ApplicationAttemptState attemptState =
+ appState.getAttempt(attemptId1);
+ Assert.assertNotNull(attemptState);
+ Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
+ attemptState.getMasterContainer().getId());
+ rm1.stop();
+
+ // start new RM
+ MockRM rm2 = new MockRM(conf, memStore);
+ rm2.start();
+
+ // verify that maxAppAttempts is set to global value
+ Assert.assertEquals(2,
+ rm2.getRMContext().getRMApps().get(app2.getApplicationId())
+ .getMaxAppAttempts());
+
+ // verify that app2 exists app1 is removed
+ Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ Assert.assertNotNull(rm2.getRMContext().getRMApps()
+ .get(app2.getApplicationId()));
+ Assert.assertNull(rm2.getRMContext().getRMApps()
+ .get(app1.getApplicationId()));
+
+ // stop the RM
+ rm2.stop();
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Fri Apr 12 23:05:28 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -192,9 +193,41 @@ public class TestResourceManager {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, -1);
try {
resourceManager.init(conf);
- fail("Exception is expected because the global max attempts is negative.");
+ fail("Exception is expected because the global max attempts" +
+ " is negative.");
} catch (YarnException e) {
// Exception is expected.
+ assertTrue("The thrown exception is not the expected one.",
+ e.getMessage().startsWith(
+ "Invalid global max attempts configuration"));
+ }
+
+ conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ try {
+ resourceManager.init(conf);
+ fail("Exception is expected because the min memory allocation is" +
+ " larger than the max memory allocation.");
+ } catch (YarnException e) {
+ // Exception is expected.
+ assertTrue("The thrown exception is not the expected one.",
+ e.getMessage().startsWith(
+ "Invalid resource scheduler memory"));
+ }
+
+ conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1);
+ try {
+ resourceManager.init(conf);
+ fail("Exception is expected because the min vcores allocation is" +
+ " larger than the max vcores allocation.");
+ } catch (YarnException e) {
+ // Exception is expected.
+ assertTrue("The thrown exception is not the expected one.",
+ e.getMessage().startsWith(
+ "Invalid resource scheduler vcores"));
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Fri Apr 12 23:05:28 2013
@@ -54,6 +54,29 @@ public class TestResourceTrackerService
private MockRM rm;
/**
+ * Test RM read NM next heartBeat Interval correctly from Configuration file,
+ * and NM get next heartBeat Interval from RM correctly
+ */
+ @Test (timeout = 5000)
+ public void testGetNextHeartBeatInterval() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "4000");
+
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+
+ NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(4000, nodeHeartbeat.getNextHeartBeatInterval());
+
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(4000, nodeHeartbeat2.getNextHeartBeatInterval());
+
+ }
+
+ /**
* Decommissioning using a pre-configured include hosts file
*/
@Test
@@ -259,7 +282,7 @@ public class TestResourceTrackerService
nodeHeartbeat = nm2.nodeHeartbeat(
new HashMap<ApplicationId, List<ContainerStatus>>(), true, -100);
- Assert.assertTrue(NodeAction.REBOOT.equals(nodeHeartbeat.getNodeAction()));
+ Assert.assertTrue(NodeAction.RESYNC.equals(nodeHeartbeat.getNodeAction()));
checkRebootedNMCount(rm, ++initialMetricCount);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java Fri Apr 12 23:05:28 2013
@@ -130,6 +130,6 @@ public class TestRMNMRPCResponseId {
nodeStatus.setResponseId(0);
response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
- Assert.assertTrue(NodeAction.REBOOT.equals(response.getNodeAction()));
+ Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Fri Apr 12 23:05:28 2013
@@ -200,13 +200,14 @@ public class TestRMAppAttemptTransitions
final String user = MockApps.newUserName();
final String queue = MockApps.newQueue();
submissionContext = mock(ApplicationSubmissionContext.class);
- when(submissionContext.getUser()).thenReturn(user);
when(submissionContext.getQueue()).thenReturn(queue);
- ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
- Resource resource = mock(Resource.class);
- when(amContainerSpec.getResource()).thenReturn(resource);
+ Resource resource = BuilderUtils.newResource(1536, 1);
+ ContainerLaunchContext amContainerSpec =
+ BuilderUtils.newContainerLaunchContext(user, null, null,
+ null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
-
+ when(submissionContext.getResource()).thenReturn(resource);
+
unmanagedAM = false;
application = mock(RMApp.class);
@@ -469,8 +470,10 @@ public class TestRMAppAttemptTransitions
// Mock the allocation of AM container
Container container = mock(Container.class);
+ Resource resource = BuilderUtils.newResource(2048, 1);
when(container.getId()).thenReturn(
BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
+ when(container.getResource()).thenReturn(resource);
Allocation allocation = mock(Allocation.class);
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));