You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/06/12 23:33:04 UTC
svn commit: r1602300 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/
hadoop-yarn/hadoop-yarn-server/hado...
Author: vinodkv
Date: Thu Jun 12 21:33:03 2014
New Revision: 1602300
URL: http://svn.apache.org/r1602300
Log:
YARN-1702. Added kill app functionality to RM web services. Contributed by Varun Vasudev.
svn merge --ignore-ancestry -c 1602298 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
- copied unchanged from r1602298, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
- copied unchanged from r1602298, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1602300&r1=1602299&r2=1602300&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Jun 12 21:33:03 2014
@@ -21,6 +21,9 @@ Release 2.5.0 - UNRELEASED
schedulers after ResourceManager Restart so as to preserve running work in
the cluster. (Jian He via vinodkv)
+ YARN-1702. Added kill app functionality to RM web services. (Varun Vasudev
+ via vinodkv)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1602300&r1=1602299&r2=1602300&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Thu Jun 12 21:33:03 2014
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.AccessControlException;
+import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
@@ -31,19 +34,27 @@ import java.util.concurrent.ConcurrentMa
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -56,6 +67,8 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -66,10 +79,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
@@ -82,7 +95,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -584,4 +596,166 @@ public class RMWebServices {
return appAttemptsInfo;
}
+
+ @GET
+ @Path("/apps/{appid}/state")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public AppState getAppState(@Context HttpServletRequest hsr,
+ @PathParam("appid") String appId) throws AuthorizationException {
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+ String userName = "";
+ if (callerUGI != null) {
+ userName = callerUGI.getUserName();
+ }
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+ "UNKNOWN", "RMWebService",
+ "Trying to get state of an absent application " + appId);
+ throw e;
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ return ret;
+ }
+
+ // can't return POJO because we can't control the status code
+ // it's always set to 200 when we need to allow it to be set
+ // to 202
+
+ @PUT
+ @Path("/apps/{appid}/state")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateAppState(AppState targetState,
+ @Context HttpServletRequest hsr, @PathParam("appid") String appId)
+ throws AuthorizationException, YarnException, InterruptedException,
+ IOException {
+
+ init();
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated";
+ throw new AuthorizationException(msg);
+ }
+
+ String userName = callerUGI.getUserName();
+ RMApp app = null;
+ try {
+ app = getRMAppForAppId(appId);
+ } catch (NotFoundException e) {
+ RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST,
+ "UNKNOWN", "RMWebService", "Trying to kill/move an absent application "
+ + appId);
+ throw e;
+ }
+
+ if (!app.getState().toString().equals(targetState.getState())) {
+ // user is attempting to change state. right we only
+ // allow users to kill the app
+
+ if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) {
+ return killApp(app, callerUGI, hsr);
+ }
+ throw new BadRequestException("Only '"
+ + YarnApplicationState.KILLED.toString()
+ + "' is allowed as a target state.");
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ return Response.status(Status.OK).entity(ret).build();
+ }
+
+ protected Response killApp(RMApp app, UserGroupInformation callerUGI,
+ HttpServletRequest hsr) throws IOException, InterruptedException {
+
+ if (app == null) {
+ throw new IllegalArgumentException("app cannot be null");
+ }
+ String userName = callerUGI.getUserName();
+ final ApplicationId appid = app.getApplicationId();
+ KillApplicationResponse resp = null;
+ try {
+ resp =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
+ @Override
+ public KillApplicationResponse run() throws IOException,
+ YarnException {
+ KillApplicationRequest req =
+ KillApplicationRequest.newInstance(appid);
+ return rm.getClientRMService().forceKillApplication(req);
+ }
+ });
+ } catch (UndeclaredThrowableException ue) {
+ // if the root cause is a permissions issue
+ // bubble that up to the user
+ if (ue.getCause() instanceof YarnException) {
+ YarnException ye = (YarnException) ue.getCause();
+ if (ye.getCause() instanceof AccessControlException) {
+ String appId = app.getApplicationId().toString();
+ String msg =
+ "Unauthorized attempt to kill appid " + appId
+ + " by remote user " + userName;
+ return Response.status(Status.FORBIDDEN).entity(msg).build();
+ } else {
+ throw ue;
+ }
+ } else {
+ throw ue;
+ }
+ }
+
+ AppState ret = new AppState();
+ ret.setState(app.getState().toString());
+
+ if (resp.getIsKillCompleted()) {
+ RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
+ "RMWebService", app.getApplicationId());
+ } else {
+ return Response.status(Status.ACCEPTED).entity(ret)
+ .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
+ }
+ return Response.status(Status.OK).entity(ret).build();
+ }
+
+ private RMApp getRMAppForAppId(String appId) {
+
+ if (appId == null || appId.isEmpty()) {
+ throw new NotFoundException("appId, " + appId + ", is empty or null");
+ }
+ ApplicationId id;
+ try {
+ id = ConverterUtils.toApplicationId(recordFactory, appId);
+ } catch (NumberFormatException e) {
+ throw new NotFoundException("appId is invalid");
+ }
+ if (id == null) {
+ throw new NotFoundException("appId is invalid");
+ }
+ RMApp app = rm.getRMContext().getRMApps().get(id);
+ if (app == null) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ return app;
+ }
+
+ private UserGroupInformation getCallerUserGroupInformation(
+ HttpServletRequest hsr) {
+
+ String remoteUser = hsr.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+
+ return callerUGI;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1602300&r1=1602299&r2=1602300&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Thu Jun 12 21:33:03 2014
@@ -1564,6 +1564,287 @@ _01_000001</amContainerLogs>
</app>
+---+
+* Cluster Application State API
+
+ With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response.
+
+ Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response.
+
+** URI
+
+-----
+ * http://<rm http address:port>/ws/v1/cluster/apps/{appid}/state
+-----
+
+** HTTP Operations Supported
+
+------
+ * GET
+ * PUT
+------
+
+** Query Parameters Supported
+
+------
+ None
+------
+
+** Elements of <appstate> object
+
+ When you make a request for the state of an app, the information returned has the following fields
+
+*---------------+--------------+-------------------------------+
+|| Item || Data Type || Description |
+*---------------+--------------+-------------------------------+
+| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" |
+*---------------+--------------+--------------------------------+
+
+
+** Response Examples
+
+ <<JSON responses>>
+
+ HTTP Request
+
+-----
+ GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+-----
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"ACCEPTED"
+}
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 202 Accepted
+Content-Type: application/json
+Transfer-Encoding: chunked
+Location: http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"ACCEPTED"
+}
++---+
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+{
+ "state":"KILLED"
+}
++---+
+
+ <<XML responses>>
+
+ HTTP Request
+
+-----
+ GET http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+-----
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 99
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>ACCEPTED</state>
+</appstate>
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 202 Accepted
+Content-Type: application/json
+Content-Length: 794
+Location: http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>ACCEPTED</state>
+</appstate>
++---+
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 200 OK
+Content-Type: application/xml
+Content-Length: 917
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ <<Unauthorized Error Response>>
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>KILLED</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 403 Unauthorized
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(6.1.26)
++---+
+
+
+ <<Bad Request Error Response>>
+
+ HTTP Request
+
+-----
+ PUT http://<rm http address:port>/ws/v1/cluster/apps/application_1399397633663_0003/state
+----
+
+ Request Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<appstate>
+ <state>RUNNING</state>
+</appstate>
++---+
+
+ Response Header:
+
++---+
+HTTP/1.1 400
+Content-Length: 295
+Content-Type: application/xml
+Server: Jetty(6.1.26)
++---+
+
+ Response Body:
+
++---+
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<RemoteException>
+ <exception>BadRequestException</exception>
+ <message>java.lang.Exception: Only 'KILLED' is allowed as a target state.</message>
+ <javaClassName>org.apache.hadoop.yarn.webapp.BadRequestException</javaClassName>
+</RemoteException>
++---+
+
* Cluster Application Attempts API
With the application attempts API, you can obtain a collection of resources that represent an application attempt. When you run a GET operation on this resource, you obtain a collection of App Attempt Objects.