You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [18/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java Tue Aug 19 23:49:39 2014
@@ -72,6 +72,9 @@ public class MetricsOverviewTable extend
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
th().$class("ui-state-default")._("Memory Reserved")._().
+ th().$class("ui-state-default")._("VCores Used")._().
+ th().$class("ui-state-default")._("VCores Total")._().
+ th().$class("ui-state-default")._("VCores Reserved")._().
th().$class("ui-state-default")._("Active Nodes")._().
th().$class("ui-state-default")._("Decommissioned Nodes")._().
th().$class("ui-state-default")._("Lost Nodes")._().
@@ -94,6 +97,9 @@ public class MetricsOverviewTable extend
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
+ td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
+ td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
+ td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
@@ -120,6 +126,9 @@ public class MetricsOverviewTable extend
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
+ th().$class("ui-state-default")._("VCores Used")._().
+ th().$class("ui-state-default")._("VCores Pending")._().
+ th().$class("ui-state-default")._("VCores Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
@@ -139,6 +148,9 @@ public class MetricsOverviewTable extend
td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
+ td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
+ td(String.valueOf(userMetrics.getPendingVirtualCores())).
+ td(String.valueOf(userMetrics.getReservedVirtualCores())).
_().
_()._();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Tue Aug 19 23:49:39 2014
@@ -75,6 +75,8 @@ class NodesPage extends RmView {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
+ th(".vcores", "VCores Used").
+ th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
@@ -127,8 +129,10 @@ class NodesPage extends RmView {
td(String.valueOf(info.getNumContainers())).
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
- td().br().$title(String.valueOf(usedMemory))._().
+ td().br().$title(String.valueOf(availableMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
+ td(String.valueOf(info.getUsedVirtualCores())).
+ td(String.valueOf(info.getAvailableVirtualCores())).
td(ni.getNodeManagerVersion()).
_();
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Tue Aug 19 23:49:39 2014
@@ -19,6 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.AccessControlException;
+import java.nio.ByteBuffer;
+import java.security.Principal;
+import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
@@ -31,31 +36,70 @@ import java.util.concurrent.ConcurrentMa
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -66,23 +110,29 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -104,6 +154,9 @@ public class RMWebServices {
private final Configuration conf;
private @Context HttpServletResponse response;
+ public final static String DELEGATION_TOKEN_HEADER =
+ "Hadoop-YARN-RM-Delegation-Token";
+
@Inject
public RMWebServices(final ResourceManager rm, Configuration conf) {
this.rm = rm;
@@ -112,11 +165,7 @@ public class RMWebServices {
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
// Check for the authorization.
- String remoteUser = hsr.getRemoteUser();
- UserGroupInformation callerUGI = null;
- if (remoteUser != null) {
- callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
- }
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI != null
&& !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
ApplicationAccessType.VIEW_APP, app.getUser(),
@@ -584,4 +633,679 @@ public class RMWebServices {
return appAttemptsInfo;
}
+
+ @GET
+ @Path("/apps/{appid}/state")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public AppState getAppState(@Context HttpServletRequest hsr,
+ @PathParam("appid") String appId) throws AuthorizationException {
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ String userName = "";
+ if (callerUGI != null) {
+ userName = callerUGI.getUserName();
+ }
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+ "UNKNOWN", "RMWebService",
+ "Trying to get state of an absent application " + appId);
+ throw e;
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ return ret;
+ }
+
+ // can't return POJO because we can't control the status code
+ // it's always set to 200 when we need to allow it to be set
+ // to 202
+
+ @PUT
+ @Path("/apps/{appid}/state")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateAppState(AppState targetState,
+ @Context HttpServletRequest hsr, @PathParam("appid") String appId)
+ throws AuthorizationException, YarnException, InterruptedException,
+ IOException {
+
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated";
+ throw new AuthorizationException(msg);
+ }
+
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
+
+ String userName = callerUGI.getUserName();
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+ "UNKNOWN", "RMWebService", "Trying to kill/move an absent application "
+ + appId);
+ throw e;
+ }
+
+ if (!app.getState().toString().equals(targetState.getState())) {
+ // user is attempting to change state. right we only
+ // allow users to kill the app
+
+ if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) {
+ return killApp(app, callerUGI, hsr);
+ }
+ throw new BadRequestException("Only '"
+ + YarnApplicationState.KILLED.toString()
+ + "' is allowed as a target state.");
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ return Response.status(Status.OK).entity(ret).build();
+ }
+
+ protected Response killApp(RMApp app, UserGroupInformation callerUGI,
+ HttpServletRequest hsr) throws IOException, InterruptedException {
+
+ if (app == null) {
+ throw new IllegalArgumentException("app cannot be null");
+ }
+ String userName = callerUGI.getUserName();
+ final ApplicationId appid = app.getApplicationId();
+ KillApplicationResponse resp = null;
+ try {
+ resp =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
+ @Override
+ public KillApplicationResponse run() throws IOException,
+ YarnException {
+ KillApplicationRequest req =
+ KillApplicationRequest.newInstance(appid);
+ return rm.getClientRMService().forceKillApplication(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ // if the root cause is a permissions issue
+ // bubble that up to the user
+ if (ue.getCause() instanceof YarnException) {
+ YarnException ye = (YarnException) ue.getCause();
+ if (ye.getCause() instanceof AccessControlException) {
+ String appId = app.getApplicationId().toString();
+ String msg =
+ "Unauthorized attempt to kill appid " + appId
+ + " by remote user " + userName;
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ } else {
+ throw ue;
+ }
+ } else {
+ throw ue;
+ }
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ if (resp.getIsKillCompleted()) {
+ RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
+ "RMWebService", app.getApplicationId());
+ } else {
+ return Response.status(Status.ACCEPTED).entity(ret)
+ .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
+ }
+ return Response.status(Status.OK).entity(ret).build();
+ }
+
+ private RMApp getRMAppForAppId(String appId) {
+
+ if (appId == null || appId.isEmpty()) {
+ throw new NotFoundException("appId, " + appId + ", is empty or null");
+ }
+ ApplicationId id;
+ try {
+ id = ConverterUtils.toApplicationId(recordFactory, appId);
+ } catch (NumberFormatException e) {
+ throw new NotFoundException("appId is invalid");
+ }
+ if (id == null) {
+ throw new NotFoundException("appId is invalid");
+ }
+ RMApp app = rm.getRMContext().getRMApps().get(id);
+ if (app == null) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ return app;
+ }
+
+ private UserGroupInformation getCallerUserGroupInformation(
+ HttpServletRequest hsr, boolean usePrincipal) {
+
+ String remoteUser = hsr.getRemoteUser();
+ if (usePrincipal) {
+ Principal princ = hsr.getUserPrincipal();
+ remoteUser = princ == null ? null : princ.getName();
+ }
+
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+
+ return callerUGI;
+ }
+
+ private boolean isStaticUser(UserGroupInformation callerUGI) {
+ String staticUser =
+ conf.get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER,
+ CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER);
+ return staticUser.equals(callerUGI.getUserName());
+ }
+
+ /**
+ * Generates a new ApplicationId which is then sent to the client
+ *
+ * @param hsr
+ * the servlet request
+ * @return Response containing the app id and the maximum resource
+ * capabilities
+ * @throws AuthorizationException
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @POST
+ @Path("/apps/new-application")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response createNewApplication(@Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException {
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ throw new AuthorizationException("Unable to obtain user name, "
+ + "user not authenticated");
+ }
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
+
+ NewApplication appId = createNewApplication();
+ return Response.status(Status.OK).entity(appId).build();
+
+ }
+
+ // reuse the code in ClientRMService to create new app
+ // get the new app id and submit app
+ // set location header with new app location
+ /**
+ * Function to submit an app to the RM
+ *
+ * @param newApp
+ * structure containing information to construct the
+ * ApplicationSubmissionContext
+ * @param hsr
+ * the servlet request
+ * @return Response containing the status code
+ * @throws AuthorizationException
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @POST
+ @Path("/apps")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
+ @Context HttpServletRequest hsr) throws AuthorizationException,
+ IOException, InterruptedException {
+
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ throw new AuthorizationException("Unable to obtain user name, "
+ + "user not authenticated");
+ }
+
+ if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
+ String msg = "The default static user cannot carry out this operation.";
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ }
+
+ ApplicationSubmissionContext appContext =
+ createAppSubmissionContext(newApp);
+ final SubmitApplicationRequest req =
+ SubmitApplicationRequest.newInstance(appContext);
+
+ try {
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<SubmitApplicationResponse>() {
+ @Override
+ public SubmitApplicationResponse run() throws IOException,
+ YarnException {
+ return rm.getClientRMService().submitApplication(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ if (ue.getCause() instanceof YarnException) {
+ throw new BadRequestException(ue.getCause().getMessage());
+ }
+ LOG.info("Submit app request failed", ue);
+ throw ue;
+ }
+
+ String url = hsr.getRequestURL() + "/" + newApp.getApplicationId();
+ return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url)
+ .build();
+ }
+
+ /**
+ * Function that actually creates the ApplicationId by calling the
+ * ClientRMService
+ *
+ * @return returns structure containing the app-id and maximum resource
+ * capabilities
+ */
+ private NewApplication createNewApplication() {
+ GetNewApplicationRequest req =
+ recordFactory.newRecordInstance(GetNewApplicationRequest.class);
+ GetNewApplicationResponse resp;
+ try {
+ resp = rm.getClientRMService().getNewApplication(req);
+ } catch (YarnException e) {
+ String msg = "Unable to create new app from RM web service";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ NewApplication appId =
+ new NewApplication(resp.getApplicationId().toString(),
+ new ResourceInfo(resp.getMaximumResourceCapability()));
+ return appId;
+ }
+
+ /**
+ * Create the actual ApplicationSubmissionContext to be submitted to the RM
+ * from the information provided by the user.
+ *
+ * @param newApp
+ * the information provided by the user
+ * @return returns the constructed ApplicationSubmissionContext
+ * @throws IOException
+ */
+ protected ApplicationSubmissionContext createAppSubmissionContext(
+ ApplicationSubmissionContextInfo newApp) throws IOException {
+
+ // create local resources and app submission context
+
+ ApplicationId appid;
+ String error =
+ "Could not parse application id " + newApp.getApplicationId();
+ try {
+ appid =
+ ConverterUtils.toApplicationId(recordFactory,
+ newApp.getApplicationId());
+ } catch (Exception e) {
+ throw new BadRequestException(error);
+ }
+ ApplicationSubmissionContext appContext =
+ ApplicationSubmissionContext.newInstance(appid,
+ newApp.getApplicationName(), newApp.getQueue(),
+ Priority.newInstance(newApp.getPriority()),
+ createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
+ newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
+ createAppSubmissionContextResource(newApp),
+ newApp.getApplicationType(),
+ newApp.getKeepContainersAcrossApplicationAttempts());
+ appContext.setApplicationTags(newApp.getApplicationTags());
+
+ return appContext;
+ }
+
+ protected Resource createAppSubmissionContextResource(
+ ApplicationSubmissionContextInfo newApp) throws BadRequestException {
+ if (newApp.getResource().getvCores() > rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) {
+ String msg = "Requested more cores than configured max";
+ throw new BadRequestException(msg);
+ }
+ if (newApp.getResource().getMemory() > rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) {
+ String msg = "Requested more memory than configured max";
+ throw new BadRequestException(msg);
+ }
+ Resource r =
+ Resource.newInstance(newApp.getResource().getMemory(), newApp
+ .getResource().getvCores());
+ return r;
+ }
+
+ /**
+ * Create the ContainerLaunchContext required for the
+ * ApplicationSubmissionContext. This function takes the user information and
+ * generates the ByteBuffer structures required by the ContainerLaunchContext
+ *
+ * @param newApp
+ * the information provided by the user
+ * @return created context
+ * @throws BadRequestException
+ * @throws IOException
+ */
+ protected ContainerLaunchContext createContainerLaunchContext(
+ ApplicationSubmissionContextInfo newApp) throws BadRequestException,
+ IOException {
+
+ // create container launch context
+
+ HashMap<String, ByteBuffer> hmap = new HashMap<String, ByteBuffer>();
+ for (Map.Entry<String, String> entry : newApp
+ .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) {
+ if (entry.getValue().isEmpty() == false) {
+ Base64 decoder = new Base64(0, null, true);
+ byte[] data = decoder.decode(entry.getValue());
+ hmap.put(entry.getKey(), ByteBuffer.wrap(data));
+ }
+ }
+
+ HashMap<String, LocalResource> hlr = new HashMap<String, LocalResource>();
+ for (Map.Entry<String, LocalResourceInfo> entry : newApp
+ .getContainerLaunchContextInfo().getResources().entrySet()) {
+ LocalResourceInfo l = entry.getValue();
+ LocalResource lr =
+ LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(),
+ l.getVisibility(), l.getSize(), l.getTimestamp());
+ hlr.put(entry.getKey(), lr);
+ }
+
+ DataOutputBuffer out = new DataOutputBuffer();
+ Credentials cs =
+ createCredentials(newApp.getContainerLaunchContextInfo()
+ .getCredentials());
+ cs.writeTokenStorageToStream(out);
+ ByteBuffer tokens = ByteBuffer.wrap(out.getData());
+
+ ContainerLaunchContext ctx =
+ ContainerLaunchContext.newInstance(hlr, newApp
+ .getContainerLaunchContextInfo().getEnvironment(), newApp
+ .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp
+ .getContainerLaunchContextInfo().getAcls());
+
+ return ctx;
+ }
+
+ /**
+ * Generate a Credentials object from the information in the CredentialsInfo
+ * object.
+ *
+ * @param credentials
+ * the CredentialsInfo provided by the user.
+ * @return
+ */
+ private Credentials createCredentials(CredentialsInfo credentials) {
+ Credentials ret = new Credentials();
+ try {
+ for (Map.Entry<String, String> entry : credentials.getTokens().entrySet()) {
+ Text alias = new Text(entry.getKey());
+ Token<TokenIdentifier> token = new Token<TokenIdentifier>();
+ token.decodeFromUrlString(entry.getValue());
+ ret.addToken(alias, token);
+ }
+ for (Map.Entry<String, String> entry : credentials.getTokens().entrySet()) {
+ Text alias = new Text(entry.getKey());
+ Base64 decoder = new Base64(0, null, true);
+ byte[] secret = decoder.decode(entry.getValue());
+ ret.addSecretKey(alias, secret);
+ }
+ } catch (IOException ie) {
+ throw new BadRequestException(
+ "Could not parse credentials data; exception message = "
+ + ie.getMessage());
+ }
+ return ret;
+ }
+
+ private UserGroupInformation createKerberosUserGroupInformation(
+ HttpServletRequest hsr) throws AuthorizationException, YarnException {
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated";
+ throw new AuthorizationException(msg);
+ }
+
+ String authType = hsr.getAuthType();
+ if (!KerberosAuthenticationHandler.TYPE.equals(authType)) {
+ String msg =
+ "Delegation token operations can only be carried out on a "
+ + "Kerberos authenticated channel";
+ throw new YarnException(msg);
+ }
+
+ callerUGI.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+ return callerUGI;
+ }
+
+ @POST
+ @Path("/delegation-token")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response postDelegationToken(DelegationToken tokenData,
+ @Context HttpServletRequest hsr) throws AuthorizationException,
+ IOException, InterruptedException, Exception {
+
+ init();
+ UserGroupInformation callerUGI;
+ try {
+ callerUGI = createKerberosUserGroupInformation(hsr);
+ } catch (YarnException ye) {
+ return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+ }
+ return createDelegationToken(tokenData, hsr, callerUGI);
+ }
+
+ @POST
+ @Path("/delegation-token/expiration")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response
+ postDelegationTokenExpiration(@Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception {
+
+ init();
+ UserGroupInformation callerUGI;
+ try {
+ callerUGI = createKerberosUserGroupInformation(hsr);
+ } catch (YarnException ye) {
+ return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+ }
+
+ DelegationToken requestToken = new DelegationToken();
+ requestToken.setToken(extractToken(hsr).encodeToUrlString());
+ return renewDelegationToken(requestToken, hsr, callerUGI);
+ }
+
+ private Response createDelegationToken(DelegationToken tokenData,
+ HttpServletRequest hsr, UserGroupInformation callerUGI)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception {
+
+ final String renewer = tokenData.getRenewer();
+ GetDelegationTokenResponse resp;
+ try {
+ resp =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
+ @Override
+ public GetDelegationTokenResponse run() throws IOException,
+ YarnException {
+ GetDelegationTokenRequest createReq =
+ GetDelegationTokenRequest.newInstance(renewer);
+ return rm.getClientRMService().getDelegationToken(createReq);
+ }
+ });
+ } catch (Exception e) {
+ LOG.info("Create delegation token request failed", e);
+ throw e;
+ }
+
+ Token<RMDelegationTokenIdentifier> tk =
+ new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
+ .getIdentifier().array(), resp.getRMDelegationToken().getPassword()
+ .array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
+ resp.getRMDelegationToken().getService()));
+ RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
+ long currentExpiration =
+ rm.getRMContext().getRMDelegationTokenSecretManager()
+ .getRenewDate(identifier);
+ DelegationToken respToken =
+ new DelegationToken(tk.encodeToUrlString(), renewer, identifier
+ .getOwner().toString(), tk.getKind().toString(), currentExpiration,
+ identifier.getMaxDate());
+ return Response.status(Status.OK).entity(respToken).build();
+ }
+
+ private Response renewDelegationToken(DelegationToken tokenData,
+ HttpServletRequest hsr, UserGroupInformation callerUGI)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception {
+
+ Token<RMDelegationTokenIdentifier> token =
+ extractToken(tokenData.getToken());
+
+ org.apache.hadoop.yarn.api.records.Token dToken =
+ BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
+ .toString(), token.getPassword(), token.getService().toString());
+ final RenewDelegationTokenRequest req =
+ RenewDelegationTokenRequest.newInstance(dToken);
+
+ RenewDelegationTokenResponse resp;
+ try {
+ resp =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {
+ @Override
+ public RenewDelegationTokenResponse run() throws IOException,
+ YarnException {
+ return rm.getClientRMService().renewDelegationToken(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ if (ue.getCause() instanceof YarnException) {
+ if (ue.getCause().getCause() instanceof InvalidToken) {
+ throw new BadRequestException(ue.getCause().getCause().getMessage());
+ } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+ return Response.status(Status.FORBIDDEN)
+ .entity(ue.getCause().getCause().getMessage()).build();
+ }
+ LOG.info("Renew delegation token request failed", ue);
+ throw ue;
+ }
+ LOG.info("Renew delegation token request failed", ue);
+ throw ue;
+ } catch (Exception e) {
+ LOG.info("Renew delegation token request failed", e);
+ throw e;
+ }
+ long renewTime = resp.getNextExpirationTime();
+
+ DelegationToken respToken = new DelegationToken();
+ respToken.setNextExpirationTime(renewTime);
+ return Response.status(Status.OK).entity(respToken).build();
+ }
+
+ // For cancelling tokens, the encoded token is passed as a header
+ // There are two reasons for this -
+ // 1. Passing a request body as part of a DELETE request is not
+ // allowed by Jetty
+ // 2. Passing the encoded token as part of the url is not ideal
+ // since urls tend to get logged and anyone with access to
+ // the logs can extract tokens which are meant to be secret
+ @DELETE
+ @Path("/delegation-token")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response cancelDelegationToken(@Context HttpServletRequest hsr)
+ throws AuthorizationException, IOException, InterruptedException,
+ Exception {
+
+ init();
+ UserGroupInformation callerUGI;
+ try {
+ callerUGI = createKerberosUserGroupInformation(hsr);
+ } catch (YarnException ye) {
+ return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+ }
+
+ Token<RMDelegationTokenIdentifier> token = extractToken(hsr);
+
+ org.apache.hadoop.yarn.api.records.Token dToken =
+ BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
+ .toString(), token.getPassword(), token.getService().toString());
+ final CancelDelegationTokenRequest req =
+ CancelDelegationTokenRequest.newInstance(dToken);
+
+ try {
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {
+ @Override
+ public CancelDelegationTokenResponse run() throws IOException,
+ YarnException {
+ return rm.getClientRMService().cancelDelegationToken(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ if (ue.getCause() instanceof YarnException) {
+ if (ue.getCause().getCause() instanceof InvalidToken) {
+ throw new BadRequestException(ue.getCause().getCause().getMessage());
+ } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+ return Response.status(Status.FORBIDDEN)
+ .entity(ue.getCause().getCause().getMessage()).build();
+ }
+ LOG.info("Renew delegation token request failed", ue);
+ throw ue;
+ }
+ LOG.info("Renew delegation token request failed", ue);
+ throw ue;
+ } catch (Exception e) {
+ LOG.info("Renew delegation token request failed", e);
+ throw e;
+ }
+
+ return Response.status(Status.OK).build();
+ }
+
+ private Token<RMDelegationTokenIdentifier> extractToken(
+ HttpServletRequest request) {
+ String encodedToken = request.getHeader(DELEGATION_TOKEN_HEADER);
+ if (encodedToken == null) {
+ String msg =
+ "Header '" + DELEGATION_TOKEN_HEADER
+ + "' containing encoded token not found";
+ throw new BadRequestException(msg);
+ }
+ return extractToken(encodedToken);
+ }
+
+ private Token<RMDelegationTokenIdentifier> extractToken(String encodedToken) {
+ Token<RMDelegationTokenIdentifier> token =
+ new Token<RMDelegationTokenIdentifier>();
+ try {
+ token.decodeFromUrlString(encodedToken);
+ } catch (Exception ie) {
+ String msg = "Could not decode encoded token";
+ throw new BadRequestException(msg);
+ }
+ return token;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java Tue Aug 19 23:49:39 2014
@@ -77,7 +77,7 @@ public class RmView extends TwoColumnLay
StringBuilder sb = new StringBuilder();
return sb
.append("[\n")
- .append("{'sType':'numeric', 'aTargets': [0]")
+ .append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': [5, 6]")
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Tue Aug 19 23:49:39 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
@@ -78,6 +79,12 @@ public class AppInfo {
protected int allocatedMB;
protected int allocatedVCores;
protected int runningContainers;
+
+ // preemption info fields
+ protected int preemptedResourceMB;
+ protected int preemptedResourceVCores;
+ protected int numNonAMContainerPreempted;
+ protected int numAMContainerPreempted;
public AppInfo() {
} // JAXB needs this
@@ -147,6 +154,17 @@ public class AppInfo {
}
}
}
+
+ // copy preemption info fields
+ RMAppMetrics appMetrics = app.getRMAppMetrics();
+ numAMContainerPreempted =
+ appMetrics.getNumAMContainersPreempted();
+ preemptedResourceMB =
+ appMetrics.getResourcePreempted().getMemory();
+ numNonAMContainerPreempted =
+ appMetrics.getNumNonAMContainersPreempted();
+ preemptedResourceVCores =
+ appMetrics.getResourcePreempted().getVirtualCores();
}
}
@@ -254,4 +272,19 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public int getPreemptedMB() {
+ return preemptedResourceMB;
+ }
+
+ public int getPreemptedVCores() {
+ return preemptedResourceVCores;
+ }
+
+ public int getNumNonAMContainersPreempted() {
+ return numNonAMContainerPreempted;
+ }
+
+ public int getNumAMContainersPreempted() {
+ return numAMContainerPreempted;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java Tue Aug 19 23:49:39 2014
@@ -37,16 +37,21 @@ public class ClusterMetricsInfo {
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
-
+
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
-
+
+ protected long reservedVirtualCores;
+ protected long availableVirtualCores;
+ protected long allocatedVirtualCores;
+
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
-
+
protected long totalMB;
+ protected long totalVirtualCores;
protected int totalNodes;
protected int lostNodes;
protected int unhealthyNodes;
@@ -68,16 +73,21 @@ public class ClusterMetricsInfo {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
+
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
-
+
+ this.reservedVirtualCores = metrics.getReservedVirtualCores();
+ this.availableVirtualCores = metrics.getAvailableVirtualCores();
+ this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
+
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
-
+
this.totalMB = availableMB + allocatedMB;
+ this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
@@ -123,6 +133,18 @@ public class ClusterMetricsInfo {
return this.allocatedMB;
}
+ public long getReservedVirtualCores() {
+ return this.reservedVirtualCores;
+ }
+
+ public long getAvailableVirtualCores() {
+ return this.availableVirtualCores;
+ }
+
+ public long getAllocatedVirtualCores() {
+ return this.allocatedVirtualCores;
+ }
+
public int getContainersAllocated() {
return this.containersAllocated;
}
@@ -134,15 +156,19 @@ public class ClusterMetricsInfo {
public int getPendingContainers() {
return this.containersPending;
}
-
+
public long getTotalMB() {
return this.totalMB;
}
+ public long getTotalVirtualCores() {
+ return this.totalVirtualCores;
+ }
+
public int getTotalNodes() {
return this.totalNodes;
}
-
+
public int getActiveNodes() {
return this.activeNodes;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java Tue Aug 19 23:49:39 2014
@@ -46,8 +46,7 @@ public class FairSchedulerInfo extends S
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
- return scheduler.getSchedulerApp(appAttemptId).
- getAppSchedulable().getFairShare().getMemory();
+ return scheduler.getSchedulerApp(appAttemptId).getFairShare().getMemory();
}
public FairSchedulerQueueInfo getRootQueueInfo() {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Tue Aug 19 23:49:39 2014
@@ -24,7 +24,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
@@ -39,9 +40,9 @@ public class FairSchedulerLeafQueueInfo
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
- Collection<AppSchedulable> apps = queue.getRunnableAppSchedulables();
- for (AppSchedulable app : apps) {
- if (app.getApp().isPending()) {
+ Collection<FSAppAttempt> apps = queue.getRunnableAppSchedulables();
+ for (FSAppAttempt app : apps) {
+ if (app.isPending()) {
numPendingApps++;
} else {
numActiveApps++;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java Tue Aug 19 23:49:39 2014
@@ -70,7 +70,7 @@ public class FairSchedulerQueueInfo {
queueName = queue.getName();
schedulingPolicy = queue.getPolicy().getName();
- clusterResources = new ResourceInfo(scheduler.getClusterCapacity());
+ clusterResources = new ResourceInfo(scheduler.getClusterResource());
usedResources = new ResourceInfo(queue.getResourceUsage());
fractionMemUsed = (float)usedResources.getMemory() /
@@ -81,7 +81,7 @@ public class FairSchedulerQueueInfo {
maxResources = new ResourceInfo(queue.getMaxShare());
maxResources = new ResourceInfo(
Resources.componentwiseMin(queue.getMaxShare(),
- scheduler.getClusterCapacity()));
+ scheduler.getClusterResource()));
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java Tue Aug 19 23:49:39 2014
@@ -43,6 +43,8 @@ public class NodeInfo {
protected int numContainers;
protected long usedMemoryMB;
protected long availMemoryMB;
+ protected long usedVirtualCores;
+ protected long availableVirtualCores;
public NodeInfo() {
} // JAXB needs this
@@ -57,6 +59,8 @@ public class NodeInfo {
this.numContainers = report.getNumContainers();
this.usedMemoryMB = report.getUsedResource().getMemory();
this.availMemoryMB = report.getAvailableResource().getMemory();
+ this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
}
this.id = id.toString();
this.rack = ni.getRackName();
@@ -83,7 +87,7 @@ public class NodeInfo {
public String getNodeHTTPAddress() {
return this.nodeHTTPAddress;
}
-
+
public void setNodeHTTPAddress(String nodeHTTPAddress) {
this.nodeHTTPAddress = nodeHTTPAddress;
}
@@ -112,4 +116,12 @@ public class NodeInfo {
return this.availMemoryMB;
}
+ public long getUsedVirtualCores() {
+ return this.usedVirtualCores;
+ }
+
+ public long getAvailableVirtualCores() {
+ return this.availableVirtualCores;
+ }
+
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java Tue Aug 19 23:49:39 2014
@@ -30,7 +30,7 @@ public class ResourceInfo {
int memory;
int vCores;
- public ResourceInfo() {
+ public ResourceInfo() {
}
public ResourceInfo(Resource res) {
@@ -50,4 +50,12 @@ public class ResourceInfo {
public String toString() {
return "<memory:" + memory + ", vCores:" + vCores + ">";
}
+
+ public void setMemory(int memory) {
+ this.memory = memory;
+ }
+
+ public void setvCores(int vCores) {
+ this.vCores = vCores;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java Tue Aug 19 23:49:39 2014
@@ -43,6 +43,9 @@ public class UserMetricsInfo {
protected long reservedMB;
protected long pendingMB;
protected long allocatedMB;
+ protected long reservedVirtualCores;
+ protected long pendingVirtualCores;
+ protected long allocatedVirtualCores;
@XmlTransient
protected boolean userMetricsAvailable;
@@ -59,7 +62,7 @@ public class UserMetricsInfo {
if (userMetrics != null) {
this.userMetricsAvailable = true;
-
+
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
@@ -70,10 +73,14 @@ public class UserMetricsInfo {
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
-
+
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
+
+ this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
+ this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
+ this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
}
}
@@ -117,6 +124,18 @@ public class UserMetricsInfo {
return this.pendingMB;
}
+ public long getReservedVirtualCores() {
+ return this.reservedVirtualCores;
+ }
+
+ public long getAllocatedVirtualCores() {
+ return this.allocatedVirtualCores;
+ }
+
+ public long getPendingVirtualCores() {
+ return this.pendingVirtualCores;
+ }
+
public int getReservedContainers() {
return this.reservedContainers;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Tue Aug 19 23:49:39 2014
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
-
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -34,6 +33,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -49,7 +49,7 @@ public class MockAM {
private volatile int responseId = 0;
private final ApplicationAttemptId attemptId;
- private final RMContext context;
+ private RMContext context;
private ApplicationMasterProtocol amRMProtocol;
private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>();
@@ -61,8 +61,10 @@ public class MockAM {
this.amRMProtocol = amRMProtocol;
this.attemptId = attemptId;
}
-
- void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol) {
+
+ public void setAMRMProtocol(ApplicationMasterProtocol amRMProtocol,
+ RMContext context) {
+ this.context = context;
this.amRMProtocol = amRMProtocol;
}
@@ -251,4 +253,22 @@ public class MockAM {
public ApplicationAttemptId getApplicationAttemptId() {
return this.attemptId;
}
+
+ public List<Container> allocateAndWaitForContainers(int nContainer,
+ int memory, MockNM nm) throws Exception {
+ // AM request for containers
+ allocate("ANY", memory, nContainer, null);
+ // kick the scheduler
+ nm.nodeHeartbeat(true);
+ List<Container> conts =
+ allocate(new ArrayList<ResourceRequest>(), null)
+ .getAllocatedContainers();
+ while (conts.size() < nContainer) {
+ nm.nodeHeartbeat(true);
+ conts.addAll(allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers());
+ Thread.sleep(500);
+ }
+ return conts;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Tue Aug 19 23:49:39 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -32,7 +31,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
@@ -88,7 +87,7 @@ public class MockNM {
return httpPort;
}
- void setResourceTrackerService(ResourceTrackerService resourceTracker) {
+ public void setResourceTrackerService(ResourceTrackerService resourceTracker) {
this.resourceTracker = resourceTracker;
}
@@ -101,19 +100,26 @@ public class MockNM {
}
public RegisterNodeManagerResponse registerNode() throws Exception {
- return registerNode(null);
+ return registerNode(null, null);
+ }
+
+ public RegisterNodeManagerResponse registerNode(
+ List<ApplicationId> runningApplications) throws Exception {
+ return registerNode(null, runningApplications);
}
public RegisterNodeManagerResponse registerNode(
- List<ContainerStatus> containerStatus) throws Exception{
+ List<NMContainerStatus> containerReports,
+ List<ApplicationId> runningApplications) throws Exception {
RegisterNodeManagerRequest req = Records.newRecord(
RegisterNodeManagerRequest.class);
req.setNodeId(nodeId);
req.setHttpPort(httpPort);
Resource resource = BuilderUtils.newResource(memory, vCores);
req.setResource(resource);
- req.setContainerStatuses(containerStatus);
+ req.setContainerStatuses(containerReports);
req.setNMVersion(version);
+ req.setRunningApplications(runningApplications);
RegisterNodeManagerResponse registrationResponse =
resourceTracker.registerNodeManager(req);
this.currentContainerTokenMasterKey =
@@ -185,4 +191,11 @@ public class MockNM {
return heartbeatResponse;
}
+ public int getMemory() {
+ return memory;
+ }
+
+ public int getvCores() {
+ return vCores;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Tue Aug 19 23:49:39 2014
@@ -21,14 +21,14 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
+import java.util.List;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
@@ -47,12 +47,15 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -61,6 +64,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -69,13 +73,19 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStartedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.junit.Assert;
@SuppressWarnings("unchecked")
public class MockRM extends ResourceManager {
@@ -144,11 +154,39 @@ public class MockRM extends ResourceMana
}
}
+ public void waitForContainerToComplete(RMAppAttempt attempt,
+ NMContainerStatus completedContainer) throws InterruptedException {
+ while (true) {
+ List<ContainerStatus> containers = attempt.getJustFinishedContainers();
+ System.out.println("Received completed containers " + containers);
+ for (ContainerStatus container : containers) {
+ if (container.getContainerId().equals(
+ completedContainer.getContainerId())) {
+ return;
+ }
+ }
+ Thread.sleep(200);
+ }
+ }
+
+ public MockAM waitForNewAMToLaunchAndRegister(ApplicationId appId, int attemptSize,
+ MockNM nm) throws Exception {
+ RMApp app = getRMContext().getRMApps().get(appId);
+ Assert.assertNotNull(app);
+ while (app.getAppAttempts().size() != attemptSize) {
+ System.out.println("Application " + appId
+ + " is waiting for AM to restart. Current has "
+ + app.getAppAttempts().size() + " attempts.");
+ Thread.sleep(200);
+ }
+ return launchAndRegisterAM(app, this, nm);
+ }
+
public void waitForState(MockNM nm, ContainerId containerId,
RMContainerState containerState) throws Exception {
RMContainer container = getResourceScheduler().getRMContainer(containerId);
int timeoutSecs = 0;
- while(container == null && timeoutSecs++ < 20) {
+ while(container == null && timeoutSecs++ < 100) {
nm.nodeHeartbeat(true);
container = getResourceScheduler().getRMContainer(containerId);
System.out.println("Waiting for container " + containerId + " to be allocated.");
@@ -329,11 +367,20 @@ public class MockRM extends ResourceMana
nm.registerNode();
return nm;
}
+
+ public MockNM registerNode(String nodeIdStr, int memory, int vCores,
+ List<ApplicationId> runningApplications) throws Exception {
+ MockNM nm =
+ new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(),
+ YarnVersionInfo.getVersion());
+ nm.registerNode(runningApplications);
+ return nm;
+ }
public void sendNodeStarted(MockNM nm) throws Exception {
RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get(
nm.getNodeId());
- node.handle(new RMNodeEvent(nm.getNodeId(), RMNodeEventType.STARTED));
+ node.handle(new RMNodeStartedEvent(nm.getNodeId(), null, null));
}
public void sendNodeLost(MockNM nm) throws Exception {
@@ -368,6 +415,13 @@ public class MockRM extends ResourceMana
throws Exception {
MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
am.waitForState(RMAppAttemptState.ALLOCATED);
+ //create and set AMRMToken
+ Token<AMRMTokenIdentifier> amrmToken =
+ this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ appAttemptId);
+ ((RMAppAttemptImpl) this.rmContext.getRMApps()
+ .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId))
+ .setAMRMToken(amrmToken);
getRMContext()
.getDispatcher()
.getEventHandler()
@@ -520,6 +574,7 @@ public class MockRM extends ResourceMana
throws Exception {
rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
RMAppAttempt attempt = app.getCurrentAppAttempt();
+ System.out.println("Launch AM " + attempt.getAppAttemptId());
nm.nodeHeartbeat(true);
MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
@@ -542,4 +597,16 @@ public class MockRM extends ResourceMana
.newInstance(appId));
return response.getApplicationReport();
}
+
+ // Explicitly reset queue metrics for testing.
+ @SuppressWarnings("static-access")
+ public void clearQueueMetrics(RMApp app) {
+ ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler())
+ .getSchedulerApplications().get(app.getApplicationId()).getQueue()
+ .getMetrics().clearQueueMetrics();
+ }
+
+ public RMActiveServices getRMActiveService() {
+ return activeServices;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java Tue Aug 19 23:49:39 2014
@@ -59,8 +59,9 @@ public class MockRMWithCustomAMLauncher
return containerManager;
}
@Override
- protected Token<AMRMTokenIdentifier> getAMRMToken() {
- Token<AMRMTokenIdentifier> amRmToken = super.getAMRMToken();
+ protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
+ Token<AMRMTokenIdentifier> amRmToken =
+ super.createAndSetAMRMToken();
InetSocketAddress serviceAddr =
getConfig().getSocketAddr(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Tue Aug 19 23:49:39 2014
@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -272,21 +273,62 @@ public class TestAMAuthorization {
client.registerApplicationMaster(request);
Assert.fail("Should fail with authorization error");
} catch (Exception e) {
- // Because there are no tokens, the request should be rejected as the
- // server side will assume we are trying simple auth.
- String expectedMessage = "";
- if (UserGroupInformation.isSecurityEnabled()) {
- expectedMessage = "Client cannot authenticate via:[TOKEN]";
+ if (isCause(AccessControlException.class, e)) {
+ // Because there are no tokens, the request should be rejected as the
+ // server side will assume we are trying simple auth.
+ String expectedMessage = "";
+ if (UserGroupInformation.isSecurityEnabled()) {
+ expectedMessage = "Client cannot authenticate via:[TOKEN]";
+ } else {
+ expectedMessage =
+ "SIMPLE authentication is not enabled. Available:[TOKEN]";
+ }
+ Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
} else {
- expectedMessage =
- "SIMPLE authentication is not enabled. Available:[TOKEN]";
+ throw e;
}
- Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
}
// TODO: Add validation of invalid authorization when there's more data in
// the AMRMToken
}
+
+ /**
+ * Identify if an expected throwable included in an exception stack. We use
+ * this because sometimes, an exception will be wrapped to another exception
+ * before thrown. Like,
+ *
+ * <pre>
+ * {@code
+ * void methodA() throws IOException {
+ * try {
+ * // something
+ * } catch (AccessControlException e) {
+ * // do process
+ * throw new IOException(e)
+ * }
+ * }
+ * </pre>
+ *
+ * So we cannot simply catch AccessControlException by using
+ * <pre>
+ * {@code
+ * try {
+ * methodA()
+ * } catch (AccessControlException e) {
+ * // do something
+ * }
+ * </pre>
+ *
+ * This method is useful in such cases.
+ */
+ private static boolean isCause(
+ Class<? extends Throwable> expected,
+ Throwable e
+ ) {
+ return (e != null)
+ && (expected.isInstance(e) || isCause(expected, e.getCause()));
+ }
private void waitForLaunchedState(RMAppAttempt attempt)
throws InterruptedException {