You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by je...@apache.org on 2013/11/13 16:48:31 UTC
svn commit: r1541565 - in
/hadoop/common/branches/branch-2.2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/
hadoop-yarn/hadoop-ya...
Author: jeagles
Date: Wed Nov 13 15:48:31 2013
New Revision: 1541565
URL: http://svn.apache.org/r1541565
Log:
YARN-1386. NodeManager mistakenly loses resources and relocalizes them (Jason Lowe via jeagles)
Modified:
hadoop/common/branches/branch-2.2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
Modified: hadoop/common/branches/branch-2.2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-yarn-project/CHANGES.txt?rev=1541565&r1=1541564&r2=1541565&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.2/hadoop-yarn-project/CHANGES.txt Wed Nov 13 15:48:31 2013
@@ -1481,7 +1481,22 @@ Release 2.0.2-alpha - 2012-09-07
YARN-138. Ensure default values for minimum/maximum container sizes is
sane. (harsh & sseth via acmurthy)
-Release 0.23.9 - UNRELEASED
+Release 0.23.10 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ YARN-1386. NodeManager mistakenly loses resources and relocalizes them
+ (Jason Lowe via jeagles)
+
+Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1541565&r1=1541564&r2=1541565&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Wed Nov 13 15:48:31 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -82,6 +83,8 @@ public class ContainerLocalizer {
public static final String WORKDIR = "work";
private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs";
private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs";
+ private static final FsPermission FILECACHE_PERMS =
+ new FsPermission((short)0710);
private final String user;
private final String appId;
@@ -363,16 +366,23 @@ public class ContainerLocalizer {
// $x/usercache/$user/filecache
Path userFileCacheDir = new Path(base, FILECACHE);
usersFileCacheDirs[i] = userFileCacheDir.toString();
- lfs.mkdir(userFileCacheDir, null, false);
+ createDir(lfs, userFileCacheDir, FILECACHE_PERMS, false);
// $x/usercache/$user/appcache/$appId
Path appBase = new Path(base, new Path(APPCACHE, appId));
// $x/usercache/$user/appcache/$appId/filecache
Path appFileCacheDir = new Path(appBase, FILECACHE);
appsFileCacheDirs[i] = appFileCacheDir.toString();
- lfs.mkdir(appFileCacheDir, null, false);
+ createDir(lfs, appFileCacheDir, FILECACHE_PERMS, false);
}
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
}
+ private static void createDir(FileContext lfs, Path dirPath,
+ FsPermission perms, boolean createParent) throws IOException {
+ lfs.mkdir(dirPath, perms, createParent);
+ if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
+ lfs.setPermission(dirPath, perms);
+ }
+ }
}
Modified: hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java?rev=1541565&r1=1541564&r2=1541565&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java (original)
+++ hadoop/common/branches/branch-2.2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java Wed Nov 13 15:48:31 2013
@@ -83,6 +83,7 @@ public class TestContainerLocalizer {
static final Log LOG = LogFactory.getLog(TestContainerLocalizer.class);
static final Path basedir =
new Path("target", TestContainerLocalizer.class.getName());
+ static final FsPermission CACHE_DIR_PERM = new FsPermission((short)0710);
static final String appUser = "yak";
static final String appId = "app_RM_0";
@@ -171,12 +172,12 @@ public class TestContainerLocalizer {
Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
// $x/usercache/$user/filecache
- verify(spylfs).mkdir(eq(privcache), isA(FsPermission.class), eq(false));
+ verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
Path appDir =
new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
// $x/usercache/$user/appcache/$appId/filecache
Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
- verify(spylfs).mkdir(eq(appcache), isA(FsPermission.class), eq(false));
+ verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
}
// verify tokens read at expected location
verify(spylfs).open(tokenPath);