You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/09/26 09:08:51 UTC
[33/50] [abbrv] hadoop git commit: YARN-3692. Allow REST API to set a
user generated message when killing an application. Contributed by Rohith
Sharma K S
YARN-3692. Allow REST API to set a user generated message when killing an application. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0372dc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0372dc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0372dc6
Branch: refs/heads/MAPREDUCE-6608
Commit: d0372dc613136910160e9d42bd5eaa0d4bde2356
Parents: 5ffd4b7
Author: Naganarasimha <na...@apache.org>
Authored: Fri Sep 23 06:30:49 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Sep 23 06:30:49 2016 +0530
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 6 ++++++
.../protocolrecords/KillApplicationRequest.java | 18 ++++++++++++++++
.../src/main/proto/yarn_service_protos.proto | 1 +
.../hadoop/yarn/client/api/YarnClient.java | 14 +++++++++++++
.../yarn/client/api/impl/YarnClientImpl.java | 22 +++++++++++++++-----
.../impl/pb/KillApplicationRequestPBImpl.java | 18 ++++++++++++++++
.../server/resourcemanager/ClientRMService.java | 20 +++++++++++++-----
.../resourcemanager/webapp/RMWebServices.java | 8 +++++--
.../resourcemanager/webapp/dao/AppState.java | 8 +++++++
.../resourcemanager/TestClientRMService.java | 7 ++++++-
.../TestRMWebServicesAppsModification.java | 4 ++++
11 files changed, 113 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 159b518..c302553 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -511,4 +511,10 @@ public class ResourceMgrDelegate extends YarnClient {
throws YarnException, IOException {
client.signalToContainer(containerId, command);
}
+
+ @Override
+ public void killApplication(ApplicationId appId, String diagnostics)
+ throws YarnException, IOException {
+ client.killApplication(appId, diagnostics);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
index 606cf4e..a7679a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.protocolrecords;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.Records;
@@ -57,4 +58,21 @@ public abstract class KillApplicationRequest {
@Public
@Stable
public abstract void setApplicationId(ApplicationId applicationId);
+
+ /**
+ * Get the <em>diagnostics</em> to which the application is being killed.
+ * @return <em>diagnostics</em> to which the application is being killed
+ */
+ @Public
+ @Unstable
+ public abstract String getDiagnostics();
+
+ /**
+ * Set the <em>diagnostics</em> to which the application is being killed.
+ * @param diagnostics <em>diagnostics</em> to which the application is being
+ * killed
+ */
+ @Public
+ @Unstable
+ public abstract void setDiagnostics(String diagnostics);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 97eaa5c..1385ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -156,6 +156,7 @@ message FailApplicationAttemptResponseProto {
message KillApplicationRequestProto {
optional ApplicationIdProto application_id = 1;
+ optional string diagnostics = 2;
}
message KillApplicationResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 619ea0d..4cac2c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -174,6 +174,20 @@ public abstract class YarnClient extends AbstractService {
/**
* <p>
+ * Kill an application identified by given ID.
+ * </p>
+ * @param applicationId {@link ApplicationId} of the application that needs to
+ * be killed
+ * @param diagnostics for killing an application.
+ * @throws YarnException in case of errors or if YARN rejects the request due
+ * to access-control restrictions.
+ * @throws IOException
+ */
+ public abstract void killApplication(ApplicationId applicationId,
+ String diagnostics) throws YarnException, IOException;
+
+ /**
+ * <p>
* Get a report of the given Application.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index b91fde0..7760521 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -405,10 +405,21 @@ public class YarnClientImpl extends YarnClient {
@Override
public void killApplication(ApplicationId applicationId)
throws YarnException, IOException {
+ killApplication(applicationId, null);
+ }
+
+ @Override
+ public void killApplication(ApplicationId applicationId, String diagnostics)
+ throws YarnException, IOException {
+
KillApplicationRequest request =
Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
+ if (diagnostics != null) {
+ request.setDiagnostics(diagnostics);
+ }
+
try {
int pollCount = 0;
long startTime = System.currentTimeMillis();
@@ -422,14 +433,15 @@ public class YarnClientImpl extends YarnClient {
}
long elapsedMillis = System.currentTimeMillis() - startTime;
- if (enforceAsyncAPITimeout() &&
- elapsedMillis >= this.asyncApiPollTimeoutMillis) {
- throw new YarnException("Timed out while waiting for application " +
- applicationId + " to be killed.");
+ if (enforceAsyncAPITimeout()
+ && elapsedMillis >= this.asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application "
+ + applicationId + " to be killed.");
}
if (++pollCount % 10 == 0) {
- LOG.info("Waiting for application " + applicationId + " to be killed.");
+ LOG.info(
+ "Waiting for application " + applicationId + " to be killed.");
}
Thread.sleep(asyncApiPollIntervalMillis);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
index db97367..0520222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
@@ -127,6 +127,24 @@ public class KillApplicationRequestPBImpl extends KillApplicationRequest {
return ((ApplicationIdPBImpl)t).getProto();
}
+ @Override
+ public String getDiagnostics() {
+ KillApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDiagnostics()) {
+ return null;
+ }
+ return (p.getDiagnostics());
+ }
+
+ @Override
+ public void setDiagnostics(String diagnostics) {
+ maybeInitBuilder();
+ if (diagnostics == null) {
+ builder.clearDiagnostics();
+ return;
+ }
+ builder.setDiagnostics(diagnostics);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 70e1863..e9bd230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -775,15 +775,25 @@ public class ClientRMService extends AbstractService implements
return KillApplicationResponse.newInstance(true);
}
- String message = "Kill application " + applicationId + " received from "
- + callerUGI;
+ StringBuilder message = new StringBuilder();
+ message.append("Application ").append(applicationId)
+ .append(" was killed by user ").append(callerUGI.getShortUserName());
+
InetAddress remoteAddress = Server.getRemoteIp();
if (null != remoteAddress) {
- message += " at " + remoteAddress.getHostAddress();
+ message.append(" at ").append(remoteAddress.getHostAddress());
+ }
+
+ String diagnostics = org.apache.commons.lang.StringUtils
+ .trimToNull(request.getDiagnostics());
+ if (diagnostics != null) {
+ message.append(" with diagnostic message: ");
+ message.append(diagnostics);
}
+
this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMAppKillByClientEvent(applicationId, message, callerUGI,
- remoteAddress));
+ .handle(new RMAppKillByClientEvent(applicationId, message.toString(),
+ callerUGI, remoteAddress));
// For UnmanagedAMs, return true so they don't retry
return KillApplicationResponse.newInstance(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 4305fd5..ef6fd06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -925,7 +925,7 @@ public class RMWebServices extends WebServices {
// allow users to kill the app
if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) {
- return killApp(app, callerUGI, hsr);
+ return killApp(app, callerUGI, hsr, targetState.getDiagnostics());
}
throw new BadRequestException("Only '"
+ YarnApplicationState.KILLED.toString()
@@ -1141,7 +1141,8 @@ public class RMWebServices extends WebServices {
}
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
- HttpServletRequest hsr) throws IOException, InterruptedException {
+ HttpServletRequest hsr, String diagnostic)
+ throws IOException, InterruptedException {
if (app == null) {
throw new IllegalArgumentException("app cannot be null");
@@ -1158,6 +1159,9 @@ public class RMWebServices extends WebServices {
YarnException {
KillApplicationRequest req =
KillApplicationRequest.newInstance(appid);
+ if (diagnostic != null) {
+ req.setDiagnostics(diagnostic);
+ }
return rm.getClientRMService().forceKillApplication(req);
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
index e8f1cc3..533200e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
public class AppState {
String state;
+ private String diagnostics;
public AppState() {
}
@@ -43,4 +44,11 @@ public class AppState {
return this.state;
}
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 2a7971e..cee9086 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -506,7 +506,8 @@ public class TestClientRMService {
@Test
public void testForceKillApplication() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- MockRM rm = new MockRM();
+ conf.setBoolean(MockRM.ENABLE_WEBAPP, true);
+ MockRM rm = new MockRM(conf);
rm.init(conf);
rm.start();
@@ -522,6 +523,8 @@ public class TestClientRMService {
KillApplicationRequest killRequest1 =
KillApplicationRequest.newInstance(app1.getApplicationId());
+ String diagnostic = "message1";
+ killRequest1.setDiagnostics(diagnostic);
KillApplicationRequest killRequest2 =
KillApplicationRequest.newInstance(app2.getApplicationId());
@@ -539,6 +542,8 @@ public class TestClientRMService {
killAttemptCount > 1);
assertEquals("Incorrect number of apps in the RM", 1,
rmService.getApplications(getRequest).getApplicationList().size());
+ assertTrue("Diagnostic message is incorrect",
+ app1.getDiagnostics().toString().contains(diagnostic));
KillApplicationResponse killResponse2 =
rmService.forceKillApplication(killRequest2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 53ef031..37e44d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -359,6 +359,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
{ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
MediaType[] contentTypes =
{ MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
+ String diagnostic = "message1";
for (String mediaType : mediaTypes) {
for (MediaType contentType : contentTypes) {
RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
@@ -366,6 +367,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
AppState targetState =
new AppState(YarnApplicationState.KILLED.toString());
+ targetState.setDiagnostics(diagnostic);
Object entity;
if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
@@ -425,6 +427,8 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
} else {
verifyAppStateXML(response, RMAppState.KILLED);
}
+ assertTrue("Diagnostic message is incorrect",
+ app.getDiagnostics().toString().contains(diagnostic));
break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org