You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/22 19:31:47 UTC
[21/51] [abbrv] hadoop git commit: YARN-5735. Make the service REST
API use the app timeout feature YARN-4205. Contributed by Jian He
YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de70e984
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de70e984
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de70e984
Branch: refs/heads/yarn-native-services
Commit: de70e984ecea1c850a33f6bb8acdb4bc7922d968
Parents: 39e18c1
Author: Gour Saha <go...@apache.org>
Authored: Fri Oct 14 17:40:51 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 10 ++++--
.../org/apache/slider/client/SliderClient.java | 33 ++++++++++----------
.../AbstractClusterBuildingActionArgs.java | 5 +++
.../slider/common/params/ActionThawArgs.java | 6 ++++
.../apache/slider/common/params/Arguments.java | 1 +
5 files changed, 36 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 21cf113..73df4a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -347,7 +347,7 @@ public class ApplicationApiService implements ApplicationApi {
if (queueName != null && queueName.trim().length() > 0) {
createArgs.queue = queueName.trim();
}
-
+ createArgs.lifetime = application.getLifetime();
return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() {
@Override
public String run(SliderClient sliderClient) throws YarnException,
@@ -1246,13 +1246,17 @@ public class ApplicationApiService implements ApplicationApi {
});
}
- private Response startSliderApplication(final String appName)
+ private Response startSliderApplication(final String appName, Application app)
throws IOException, YarnException, InterruptedException {
return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() {
@Override
public Response run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
ActionThawArgs thawArgs = new ActionThawArgs();
+ if (app.getLifetime() == null) {
+ app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+ }
+ thawArgs.lifetime = app.getLifetime();
int returnCode = sliderClient.actionThaw(appName, thawArgs);
if (returnCode == 0) {
logger.info("Successfully started application {}", appName);
@@ -1344,7 +1348,7 @@ public class ApplicationApiService implements ApplicationApi {
try {
int livenessCheck = getSliderList(appName);
if (livenessCheck != 0) {
- return startSliderApplication(appName);
+ return startSliderApplication(appName, updateAppData);
} else {
logger.info("Application {} is already running", appName);
ApplicationStatus applicationStatus = new ApplicationStatus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe4f1d2..2840c4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -734,7 +735,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
throw e;
}
- return startCluster(clustername, createArgs);
+ return startCluster(clustername, createArgs, createArgs.lifetime);
}
@Override
@@ -1960,14 +1961,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*
* @param clustername name of the cluster.
* @param launchArgs launch arguments
+ * @param lifetime
* @return the exit code
* @throws YarnException
* @throws IOException
*/
- protected int startCluster(String clustername,
- LaunchArgsAccessor launchArgs) throws
- YarnException,
- IOException {
+ protected int startCluster(String clustername, LaunchArgsAccessor launchArgs,
+ long lifetime) throws YarnException, IOException {
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@@ -1975,7 +1975,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
LaunchedApplication launchedApplication =
launchApplication(clustername, clusterDirectory, instanceDefinition,
- serviceArgs.isDebug());
+ serviceArgs.isDebug(), lifetime);
if (launchArgs.getOutputFile() != null) {
// output file has been requested. Get the app report and serialize it
@@ -2044,9 +2044,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
protected AppMasterLauncher setupAppMasterLauncher(String clustername,
- Path clusterDirectory,
- AggregateConf instanceDefinition,
- boolean debugAM)
+ Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM,
+ long lifetime)
throws YarnException, IOException{
deployedClusterName = clustername;
validateClusterName(clustername);
@@ -2119,7 +2118,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ApplicationId appId = amLauncher.getApplicationId();
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
-
+ // set lifetime in submission context;
+ Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
+ appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+ amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
amLauncher.setMaxAppAttempts(maxAppAttempts);
@@ -2383,20 +2385,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param clusterDirectory cluster dir
* @param instanceDefinition the instance definition
* @param debugAM enable debug AM options
+ * @param lifetime
* @return the launched application
* @throws YarnException
* @throws IOException
*/
- public LaunchedApplication launchApplication(String clustername,
- Path clusterDirectory,
- AggregateConf instanceDefinition,
- boolean debugAM)
+ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory,
+ AggregateConf instanceDefinition, boolean debugAM, long lifetime)
throws YarnException, IOException {
AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername,
clusterDirectory,
instanceDefinition,
- debugAM);
+ debugAM, lifetime);
applicationId = amLauncher.getApplicationId();
log.info("Submitting application {}", applicationId);
@@ -3254,7 +3255,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
verifyNoLiveClusters(clustername, "Start");
//start the cluster
- return startCluster(clustername, thaw);
+ return startCluster(clustername, thaw, thaw.lifetime);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 2a5eedc..3cb75e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -102,6 +102,11 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Queue to submit the application")
public String queue;
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the application since application started at"
+ + " running state")
+ public long lifetime;
+
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
index b43a14e..2bd856f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
@@ -18,6 +18,7 @@
package org.apache.slider.common.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@@ -43,6 +44,11 @@ public class ActionThawArgs extends AbstractActionArgs implements
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the application since application started at"
+ + " running state")
+ public long lifetime;
+
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index aec4e26..cbf7e59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -103,6 +103,7 @@ public interface Arguments {
String ARG_PRINCIPAL = "--principal";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
+ String ARG_LIFETIME = "--lifetime";
String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCES = "--resources";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org