You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/09/28 02:55:22 UTC
[31/50] [abbrv] hadoop git commit: YARN-6871. Add additional
deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8facf1f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8facf1f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8facf1f9
Branch: refs/heads/YARN-5734
Commit: 8facf1f976d7e12a846f12baabf54be1b7a49f9d
Parents: dd37877
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 27 14:37:32 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 27 14:37:32 2017 +0530
----------------------------------------------------------------------
.../resourcemanager/webapp/DeSelectFields.java | 11 +-
.../resourcemanager/webapp/dao/AppInfo.java | 108 +++++++++++++------
.../webapp/TestRMWebServicesApps.java | 77 ++++++++++++-
3 files changed, 160 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
index 258bbfa..c991766 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java
@@ -91,7 +91,16 @@ public class DeSelectFields {
* <code>RESOURCE_REQUESTS</code> is the first
* supported type from YARN-6280.
*/
- RESOURCE_REQUESTS("resourceRequests");
+ RESOURCE_REQUESTS("resourceRequests"),
+ /**
+ * <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION,
+ * RESOURCE_INFO</code> are additionally supported parameters added in
+ * YARN-6871.
+ */
+ TIMEOUTS("timeouts"),
+ APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"),
+ AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"),
+ RESOURCE_INFO("resourceInfo");
private final String literals;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 236c467..40d7b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -123,7 +123,7 @@ public class AppInfo {
protected String amNodeLabelExpression;
protected ResourcesInfo resourceInfo = null;
- protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo();
+ private AppTimeoutsInfo timeouts;
public AppInfo() {
} // JAXB needs this
@@ -247,47 +247,87 @@ public class AppInfo {
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
- amNodeLabelExpression = (unmanagedApplication) ? null
- : app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "appNodeLabelExpression", objects
+ * pertaining to the appNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) {
+ appNodeLabelExpression =
+ app.getApplicationSubmissionContext().getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "amNodeLabelExpression", objects
+ * pertaining to the amNodeLabelExpression are not returned. By default,
+ * this is not skipped. (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) {
+ amNodeLabelExpression = (unmanagedApplication) ?
+ null :
+ app.getAMResourceRequests().get(0).getNodeLabelExpression();
+ }
+ /*
+ * When the deSelects parameter contains "resourceInfo", ResourceInfo
+ * objects are not returned. Default behavior is no skipping. (YARN-6871)
+ */
// Setting partition based resource usage of application
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof CapacityScheduler) {
- RMAppAttempt attempt = app.getCurrentAppAttempt();
- if (null != attempt) {
- FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
- .getApplicationAttempt(attempt.getAppAttemptId());
- resourceInfo = null != ficaAppAttempt
- ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage())
- : null;
+ if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof CapacityScheduler) {
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ if (null != attempt) {
+ FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(attempt.getAppAttemptId());
+ resourceInfo = null != ficaAppAttempt ?
+ new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) :
+ null;
+ }
}
}
- Map<ApplicationTimeoutType, Long> applicationTimeouts =
- app.getApplicationTimeouts();
- if (applicationTimeouts.isEmpty()) {
- // If application is not set timeout, lifetime should be sent as default
- // with expiryTime=UNLIMITED and remainingTime=-1
- AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
- timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
- timeouts.add(timeoutInfo);
- } else {
- for (Map.Entry<ApplicationTimeoutType, Long> entry : app
- .getApplicationTimeouts().entrySet()) {
- AppTimeoutInfo timeout = new AppTimeoutInfo();
- timeout.setTimeoutType(entry.getKey());
- long timeoutInMillis = entry.getValue().longValue();
- timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
- if (app.isAppInCompletedStates()) {
- timeout.setRemainingTime(0);
- } else {
- timeout.setRemainingTime(Math
- .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ /*
+ * When the deSelects parameter contains "appTimeouts", objects pertaining
+ * to app timeouts are not returned. By default, this is not skipped.
+ * (YARN-6871)
+ */
+ if (!deSelects.contains(DeSelectType.TIMEOUTS)) {
+ Map<ApplicationTimeoutType, Long> applicationTimeouts =
+ app.getApplicationTimeouts();
+ if (applicationTimeouts.isEmpty()) {
+ // If application is not set timeout, lifetime should be sent
+ // as default with expiryTime=UNLIMITED and remainingTime=-1
+ AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+ timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+ timeouts = new AppTimeoutsInfo();
+ timeouts.add(timeoutInfo);
+ } else {
+ for (Map.Entry<ApplicationTimeoutType, Long> entry : app
+ .getApplicationTimeouts().entrySet()) {
+ AppTimeoutInfo timeout = new AppTimeoutInfo();
+ timeout.setTimeoutType(entry.getKey());
+ long timeoutInMillis = entry.getValue().longValue();
+ timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis));
+ if (app.isAppInCompletedStates()) {
+ timeout.setRemainingTime(0);
+ } else {
+ timeout.setRemainingTime(Math.max(
+ (timeoutInMillis - System.currentTimeMillis()) / 1000, 0));
+ }
+ timeouts.add(timeout);
}
- timeouts.add(timeout);
}
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8facf1f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 07f74a3..f0704ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1113,8 +1113,83 @@ public class TestRMWebServicesApps extends JerseyTestBase {
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
- assertTrue("resource requests shouldn't exits",
+ assertTrue("resource requests shouldn't exist",
!app.has("resourceRequests"));
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AMNodeLabelExpression shouldn't exist",
+ !app.has("amNodeLabelExpression"));
+
+ params.clear();
+ params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Timeouts shouldn't exist", !app.has("timeouts"));
+ rm.stop();
+
+ params.clear();
+ params.add("deSelects",
+ DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("AppNodeLabelExpression shouldn't exist",
+ !app.has("appNodeLabelExpression"));
+ rm.stop();
+
+ params.clear();
+ params
+ .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString());
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps").queryParams(params)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ app = array.getJSONObject(0);
+ assertTrue("Resource info shouldn't exist", !app.has("resourceInfo"));
rm.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org