You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:38 UTC
[06/50] [abbrv] hadoop git commit: YARN-2355. MAX_APP_ATTEMPTS_ENV
may no longer be a useful env var for a container (Darrell Taylor via aw)
YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e06811e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e06811e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e06811e6
Branch: refs/heads/YARN-2928
Commit: e06811e689447cb30418ae66f1e39fe9ff2ecbb3
Parents: 0cd01a1
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:40:56 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/ApplicationConstants.java | 50 +++++++++-----------
.../resourcemanager/amlauncher/AMLauncher.java | 29 +++++-------
.../TestApplicationMasterLauncher.java | 7 ++-
4 files changed, 41 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c8dbb08..54b33bc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -136,6 +136,9 @@ Trunk - Unreleased
YARN-2428. LCE default banned user list should have yarn (Varun
Saxena via aw)
+ YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var
+ for a container (Darrell Taylor via aw)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 42464da..d75297a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.util.Shell;
/**
* This is the API for the applications comprising of constants that YARN sets
* up for the applications and the containers.
- *
+ *
* TODO: Investigate the semantics and security of each cross-boundary refs.
*/
@Public
@Evolving
public interface ApplicationConstants {
-
+
/**
* The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
* only
@@ -48,11 +48,11 @@ public interface ApplicationConstants {
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
/**
- * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
* ApplicationMaster's environment only. This states that for all non-relative
* web URLs in the app masters web UI what base should they have.
*/
- public static final String APPLICATION_WEB_PROXY_BASE_ENV =
+ public static final String APPLICATION_WEB_PROXY_BASE_ENV =
"APPLICATION_WEB_PROXY_BASE";
/**
@@ -99,15 +99,9 @@ public interface ApplicationConstants {
public static final String STDOUT = "stdout";
/**
- * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment
- * only
- */
- public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS";
-
- /**
* Environment for Applications.
- *
- * Some of the environment variables for applications are <em>final</em>
+ *
+ * Some of the environment variables for applications are <em>final</em>
* i.e. they cannot be modified by the applications.
*/
public enum Environment {
@@ -116,76 +110,76 @@ public interface ApplicationConstants {
* Final, non-modifiable.
*/
USER("USER"),
-
+
/**
* $LOGNAME
* Final, non-modifiable.
*/
LOGNAME("LOGNAME"),
-
+
/**
* $HOME
* Final, non-modifiable.
*/
HOME("HOME"),
-
+
/**
* $PWD
* Final, non-modifiable.
*/
PWD("PWD"),
-
+
/**
* $PATH
*/
PATH("PATH"),
-
+
/**
* $SHELL
*/
SHELL("SHELL"),
-
+
/**
* $JAVA_HOME
*/
JAVA_HOME("JAVA_HOME"),
-
+
/**
* $CLASSPATH
*/
CLASSPATH("CLASSPATH"),
-
+
/**
* $APP_CLASSPATH
*/
APP_CLASSPATH("APP_CLASSPATH"),
-
+
/**
* $LD_LIBRARY_PATH
*/
LD_LIBRARY_PATH("LD_LIBRARY_PATH"),
-
+
/**
* $HADOOP_CONF_DIR
* Final, non-modifiable.
*/
HADOOP_CONF_DIR("HADOOP_CONF_DIR"),
-
+
/**
* $HADOOP_COMMON_HOME
*/
HADOOP_COMMON_HOME("HADOOP_COMMON_HOME"),
-
+
/**
* $HADOOP_HDFS_HOME
*/
HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"),
-
+
/**
* $MALLOC_ARENA_MAX
*/
MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"),
-
+
/**
* $HADOOP_YARN_HOME
*/
@@ -240,11 +234,11 @@ public interface ApplicationConstants {
private Environment(String variable) {
this.variable = variable;
}
-
+
public String key() {
return variable;
}
-
+
public String toString() {
return variable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index f62ee50..42bf579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -80,10 +80,10 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
-
+
@SuppressWarnings("rawtypes")
private final EventHandler handler;
-
+
public AMLauncher(RMContext rmContext, RMAppAttempt application,
AMLauncherEventType eventType, Configuration conf) {
this.application = application;
@@ -93,20 +93,20 @@ public class AMLauncher implements Runnable {
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
}
-
+
private void connect() throws IOException {
ContainerId masterContainerID = masterContainer.getId();
-
+
containerMgrProxy = getContainerMgrProxy(masterContainerID);
}
-
+
private void launch() throws IOException, YarnException {
connect();
ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext =
application.getSubmissionContext();
LOG.info("Setting up container " + masterContainer
- + " for AM " + application.getAppAttemptId());
+ + " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
createAMContainerLaunchContext(applicationContext, masterContainerID);
@@ -130,7 +130,7 @@ public class AMLauncher implements Runnable {
+ application.getAppAttemptId());
}
}
-
+
private void cleanup() throws IOException, YarnException {
connect();
ContainerId containerId = masterContainer.getId();
@@ -188,17 +188,17 @@ public class AMLauncher implements Runnable {
ContainerId containerID) throws IOException {
// Construct the actual Container
- ContainerLaunchContext container =
+ ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
LOG.info("Command to launch container "
+ containerID
+ " : "
+ StringUtils.arrayToString(container.getCommands().toArray(
new String[0])));
-
+
// Finalize the container
setupTokens(container, containerID);
-
+
return container;
}
@@ -210,7 +210,7 @@ public class AMLauncher implements Runnable {
Map<String, String> environment = container.getEnvironment();
environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
application.getWebProxyBase());
- // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
+ // Set AppSubmitTime to be consumable by the AM.
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
environment.put(
@@ -218,9 +218,6 @@ public class AMLauncher implements Runnable {
String.valueOf(rmContext.getRMApps()
.get(applicationId)
.getSubmitTime()));
- environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV,
- String.valueOf(rmContext.getRMApps().get(
- applicationId).getMaxAppAttempts()));
// Set flow context info
for (String tag :
rmContext.getRMApps().get(applicationId).getApplicationTags()) {
@@ -267,7 +264,7 @@ public class AMLauncher implements Runnable {
((RMAppAttemptImpl)application).setAMRMToken(amrmToken);
return amrmToken;
}
-
+
@SuppressWarnings("unchecked")
public void run() {
switch (eventType) {
@@ -297,7 +294,7 @@ public class AMLauncher implements Runnable {
sb.append(" is not handled by this NodeManager");
if (!e.getMessage().contains(sb.toString())) {
// Ignoring if container is already killed by Node Manager.
- LOG.info("Error cleaning master ", e);
+ LOG.info("Error cleaning master ", e);
}
}
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 9a4395e..d4f8e93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -107,8 +107,7 @@ public class TestApplicationMasterLauncher {
nmHostAtContainerManager = tokenId.getNmHostAddress();
submitTimeAtContainerManager =
Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
- maxAppAttempts =
- Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
+ maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS;
return StartContainersResponse.newInstance(
new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
new HashMap<ContainerId, SerializedException>());
@@ -184,8 +183,8 @@ public class TestApplicationMasterLauncher {
am.waitForState(RMAppAttemptState.FINISHED);
rm.stop();
}
-
-
+
+
@SuppressWarnings("unused")
@Test(timeout = 100000)
public void testallocateBeforeAMRegistration() throws Exception {