You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/06/16 01:12:10 UTC
svn commit: r1493430 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoo...
Author: vinodkv
Date: Sat Jun 15 23:12:10 2013
New Revision: 1493430
URL: http://svn.apache.org/r1493430
Log:
YARN-781. Exposing LOGDIR in all containers' environment which should be used by containers for logging purposes. Contributed by Jian He.
svn merge --ignore-ancestry -c 1493428 ../../trunk/
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1493430&r1=1493429&r2=1493430&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Sat Jun 15 23:12:10 2013
@@ -132,6 +132,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-398. Make it possible to specify hard locality constraints in resource
requests for CapacityScheduler. (acmurthy)
+ YARN-781. Exposing LOGDIR in all containers' environment which should be used
+ by containers for logging purposes. (Jian He via vinodkv)
+
IMPROVEMENTS
YARN-365. Change NM heartbeat handling to not generate a scheduler event
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1493430&r1=1493429&r2=1493430&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Sat Jun 15 23:12:10 2013
@@ -177,7 +177,15 @@ public interface ApplicationConstants {
* $LOCAL_DIRS
* Final, exported by NodeManager and non-modifiable by users.
*/
- LOCAL_DIRS("LOCAL_DIRS");
+ LOCAL_DIRS("LOCAL_DIRS"),
+
+ /**
+ * $LOG_DIRS
+ * Final, exported by NodeManager and non-modifiable by users.
+ * Comma separate list of directories that the container should use for
+ * logging.
+ */
+ LOG_DIRS("LOG_DIRS");
private final String variable;
private Environment(String variable) {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1493430&r1=1493429&r2=1493430&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Sat Jun 15 23:12:10 2013
@@ -132,9 +132,10 @@ public class ContainerLaunch implements
// Before the container script gets written out.
List<String> newCmds = new ArrayList<String>(command.size());
String appIdStr = app.getAppId().toString();
+ String relativeContainerLogDir = ContainerLaunch
+ .getRelativeContainerLogDir(appIdStr, containerIdStr);
Path containerLogDir =
- dirsHandler.getLogPathForWrite(ContainerLaunch
- .getRelativeContainerLogDir(appIdStr, containerIdStr), false);
+ dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
for (String str : command) {
// TODO: Should we instead work via symlinks without this grammar?
newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
@@ -189,6 +190,11 @@ public class ContainerLaunch implements
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
+ List<String> containerLogDirs = new ArrayList<String>();
+ for( String logDir : logDirs) {
+ containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
+ }
+
if (!dirsHandler.areDisksHealthy()) {
ret = ContainerExitStatus.DISKS_FAILED;
throw new IOException("Most of the disks failed. "
@@ -215,7 +221,8 @@ public class ContainerLaunch implements
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
- sanitizeEnv(environment, containerWorkDir, appDirs, localResources);
+ sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
+ localResources);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -543,9 +550,9 @@ public class ContainerLaunch implements
}
}
- public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs, Map<Path, List<String>> resources)
- throws IOException {
+ public void sanitizeEnv(Map<String, String> environment, Path pwd,
+ List<Path> appDirs, List<String> containerLogDirs,
+ Map<Path, List<String>> resources) throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -565,6 +572,9 @@ public class ContainerLaunch implements
environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
+ environment.put(Environment.LOG_DIRS.name(),
+ StringUtils.join(",", containerLogDirs));
+
putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
putEnvIfNotNull(environment,
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1493430&r1=1493429&r2=1493430&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Sat Jun 15 23:12:10 2013
@@ -250,6 +250,8 @@ public class TestContainerLaunch extends
// Now verify the contents of the file
List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> logDirs = dirsHandler.getLogDirs();
+
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
for (String localDir : localDirs) {
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
@@ -257,6 +259,12 @@ public class TestContainerLaunch extends
Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
appDirs.add(new Path(appsdir, appId.toString()));
}
+ List<String> containerLogDirs = new ArrayList<String>();
+ String relativeContainerLogDir = ContainerLaunch
+ .getRelativeContainerLogDir(appId.toString(), cId.toString());
+ for(String logDir : logDirs){
+ containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
+ }
BufferedReader reader =
new BufferedReader(new FileReader(processStartFile));
Assert.assertEquals(cId.toString(), reader.readLine());
@@ -276,6 +284,9 @@ public class TestContainerLaunch extends
.getEnvironment().get(Environment.NM_HTTP_PORT.name()));
Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext
.getEnvironment().get(Environment.LOCAL_DIRS.name()));
+ Assert.assertEquals(StringUtils.join(",", containerLogDirs),
+ containerLaunchContext.getEnvironment().get(Environment.LOG_DIRS.name()));
+
// Get the pid of the process
String pid = reader.readLine().trim();
// No more lines