You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by go...@apache.org on 2016/12/16 20:04:42 UTC
hadoop git commit: YARN-5740. Add a new field in Slider status output
- lifetime (remaining). Contributed by Jian He
Repository: hadoop
Updated Branches:
refs/heads/yarn-native-services c22be358a -> de82afc8a
YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de82afc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de82afc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de82afc8
Branch: refs/heads/yarn-native-services
Commit: de82afc8a53c296a6a24a0f0d2024a4dd5ea5390
Parents: c22be35
Author: Gour Saha <go...@apache.org>
Authored: Fri Dec 16 12:03:51 2016 -0800
Committer: Gour Saha <go...@apache.org>
Committed: Fri Dec 16 12:03:51 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 82 ++++++++++++--------
.../org/apache/slider/client/SliderClient.java | 62 +++++++++++----
.../slider/common/params/ActionStatusArgs.java | 4 +
3 files changed, 103 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de82afc8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index c4f5d43..b11da2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -52,6 +52,8 @@ import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.services.api.ApplicationApi;
@@ -771,7 +773,7 @@ public class ApplicationApiService implements ApplicationApi {
// Get all applications in a specific state - lighter projection. For full
// detail, call getApplication on a specific app.
- Set<String> applications;
+ Set<ApplicationReport> applications;
try {
if (StringUtils.isNotEmpty(state)) {
ApplicationStatus appStatus = new ApplicationStatus();
@@ -793,13 +795,12 @@ public class ApplicationApiService implements ApplicationApi {
Set<Application> apps = new HashSet<Application>();
if (applications.size() > 0) {
try {
- for (String app : applications) {
+ for (ApplicationReport app : applications) {
Application application = new Application();
- // TODO: Need to get lifetime, launch-time and privileged container
- // status from YARN
- application.setLifetime(null);
- application.setLaunchTime(new Date());
- application.setName(app);
+ application.setLifetime(app.getApplicationTimeouts().get(
+ ApplicationTimeoutType.LIFETIME).getRemainingTime());
+ application.setLaunchTime(new Date(app.getStartTime()));
+ application.setName(app.getName());
// Containers not required, setting to null to avoid empty list
application.setContainers(null);
apps.add(application);
@@ -930,9 +931,7 @@ public class ApplicationApiService implements ApplicationApi {
app.setLaunchTime(appStatus.get("createTime") == null ? null
: new Date(appStatus.get("createTime").getAsLong()));
- // lifetime - set it to unlimited for now
- // TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
- app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+ app.setLifetime(queryLifetime(appName));
// Quicklinks
Map<String, String> appQuicklinks = new HashMap<>();
@@ -1062,6 +1061,24 @@ public class ApplicationApiService implements ApplicationApi {
return object.get(key) == null ? null : object.get(key).getAsJsonObject();
}
+ private long queryLifetime(String appName) {
+ try {
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<Long>() {
+ @Override
+ public Long run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ ApplicationReport report = sliderClient.findInstance(appName);
+ return report.getApplicationTimeouts()
+ .get(ApplicationTimeoutType.LIFETIME).getRemainingTime();
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Error when querying lifetime for " + appName, e);
+ return DEFAULT_UNLIMITED_LIFETIME;
+ }
+ }
+
private JsonObject getSliderApplicationStatus(final String appName)
throws IOException, YarnException, InterruptedException {
@@ -1142,36 +1159,37 @@ public class ApplicationApiService implements ApplicationApi {
});
}
- private Set<String> getSliderApplications(final String state)
+ private Set<ApplicationReport> getSliderApplications(final String state)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(false, state);
}
- private Set<String> getSliderApplications(final boolean liveOnly)
+ private Set<ApplicationReport> getSliderApplications(final boolean liveOnly)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(liveOnly, null);
}
- private Set<String> getSliderApplications(final boolean liveOnly,
- final String state) throws IOException, YarnException,
- InterruptedException {
- return invokeSliderClientRunnable(new SliderClientContextRunnable<Set<String>>() {
- @Override
- public Set<String> run(SliderClient sliderClient) throws YarnException,
- IOException, InterruptedException {
- Set<String> apps;
- ActionListArgs listArgs = new ActionListArgs();
- if (liveOnly) {
- apps = sliderClient.getApplicationList(null);
- } else if (StringUtils.isNotEmpty(state)) {
- listArgs.state = state;
- apps = sliderClient.getApplicationList(null, listArgs);
- } else {
- apps = sliderClient.getApplicationList(null, listArgs);
- }
- return apps;
- }
- });
+ private Set<ApplicationReport> getSliderApplications(final boolean liveOnly,
+ final String state)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<Set<ApplicationReport>>() {
+ @Override
+ public Set<ApplicationReport> run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ Set<ApplicationReport> apps;
+ ActionListArgs listArgs = new ActionListArgs();
+ if (liveOnly) {
+ apps = sliderClient.getApplicationList(null);
+ } else if (StringUtils.isNotEmpty(state)) {
+ listArgs.state = state;
+ apps = sliderClient.getApplicationList(null, listArgs);
+ } else {
+ apps = sliderClient.getApplicationList(null, listArgs);
+ }
+ return apps;
+ }
+ });
}
@DELETE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de82afc8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef45d10..1c126ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -178,6 +179,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
@@ -185,6 +187,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
+import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -2706,11 +2709,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
- Set<String> appInstances = getApplicationList(clustername, args);
- // getApplicationList never returns null
- return !appInstances.isEmpty() ? EXIT_SUCCESS
- : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS
- : EXIT_FALSE);
+ Set<ApplicationReport> appInstances = getApplicationList(clustername, args);
+ if (!appInstances.isEmpty()) {
+ return EXIT_SUCCESS;
+ } else {
+ return EXIT_FALSE;
+ }
}
/**
@@ -2723,8 +2727,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException
* @throws YarnException
*/
- public Set<String> getApplicationList(String clustername) throws IOException,
- YarnException {
+ public Set<ApplicationReport> getApplicationList(String clustername)
+ throws IOException, YarnException {
ActionListArgs args = new ActionListArgs();
args.live = true;
return getApplicationList(clustername, args);
@@ -2743,8 +2747,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws UnknownApplicationInstanceException
* if a specific instance was named but it was not found
*/
- public Set<String> getApplicationList(String clustername, ActionListArgs args)
- throws IOException, YarnException {
+ public Set<ApplicationReport> getApplicationList(String clustername,
+ ActionListArgs args) throws IOException, YarnException {
if (args.help) {
actionHelp(ACTION_LIST);
// the above call throws an exception so the return is not really required
@@ -2830,13 +2834,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
// at this point there is either the entire list or a stripped down instance
- Set<String> listedInstances = new HashSet<String>();
+ Set<ApplicationReport> listedInstances = new HashSet<ApplicationReport>();
for (String name : persistentInstances.keySet()) {
ApplicationReport report = reportMap.get(name);
if (!listOnlyInState || report != null) {
// list the details if all were requested, or the filtering contained
// a report
- listedInstances.add(name);
+ listedInstances.add(report);
// containers will be non-null when only one instance is requested
String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
@@ -3055,7 +3059,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
- private ApplicationReport findInstance(String appname)
+ public ApplicationReport findInstance(String appname)
throws YarnException, IOException {
return yarnAppListClient.findInstance(appname);
}
@@ -3106,6 +3110,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException {
+ if (statusArgs.lifetime) {
+ queryAndPrintLifetime(clustername);
+ return EXIT_SUCCESS;
+ }
+
ClusterDescription status = verifyAndGetClusterDescription(clustername);
String outfile = statusArgs.getOutput();
if (outfile == null) {
@@ -3122,6 +3131,32 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return verifyAndGetClusterDescription(clustername).toJsonString();
}
+ private void queryAndPrintLifetime(String appName)
+ throws YarnException, IOException {
+ ApplicationReport appReport = findInstance(appName);
+ if (appReport == null) {
+ throw new YarnException("No application found for " + appName);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter timeoutStr =
+ new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
+ try {
+ ApplicationTimeout lifetime = appReport.getApplicationTimeouts()
+ .get(ApplicationTimeoutType.LIFETIME);
+ if (lifetime.getRemainingTime() == -1L) {
+ timeoutStr.append(appName + " has no lifetime configured.");
+ } else {
+ timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME);
+ timeoutStr.print(" expires at : " + lifetime.getExpiryTime());
+ timeoutStr.println(
+ ".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds");
+ }
+ System.out.println(baos.toString("UTF-8"));
+ } finally {
+ timeoutStr.close();
+ }
+ }
+
private ClusterDescription verifyAndGetClusterDescription(String clustername)
throws YarnException, IOException {
verifyBindingsDefined();
@@ -3547,7 +3582,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException
*/
@VisibleForTesting
- public List<ApplicationReport> getApplications() throws YarnException, IOException {
+ public List<ApplicationReport> getApplications()
+ throws YarnException, IOException {
return yarnClient.getApplications();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de82afc8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
index 00178df..6fbd96d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
@@ -35,6 +35,10 @@ public class ActionStatusArgs extends AbstractActionArgs {
description = "Output file for the status information")
public String output;
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Lifetime of the application from the time of request")
+ public boolean lifetime;
+
public String getOutput() {
return output;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org