You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2014/11/03 21:40:00 UTC
git commit: YARN-2730. DefaultContainerExecutor runs only one
localizer at a time. Contributed by Siqi Li (cherry picked from commit
6157ace5475fff8d2513fd3cd99134b532b0b406)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c8648bd27 -> 8091ea54d
YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li
(cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8091ea54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8091ea54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8091ea54
Branch: refs/heads/branch-2
Commit: 8091ea54d5e5c47da95ac1b1d921134399a5f066
Parents: c8648bd
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Nov 3 20:37:47 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Nov 3 20:39:14 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/DefaultContainerExecutor.java | 19 ++++++++++++-------
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8091ea54/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9ae57e4..56359e5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -809,6 +809,9 @@ Release 2.6.0 - UNRELEASED
YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun Vasudev
via zjshen)
+ YARN-2730. DefaultContainerExecutor runs only one localizer at a time
+ (Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8091ea54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 834b138..cc2de99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -94,7 +94,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
@Override
- public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
+ public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
@@ -102,10 +102,6 @@ public class DefaultContainerExecutor extends ContainerExecutor {
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
- ContainerLocalizer localizer =
- new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
- RecordFactoryProvider.getRecordFactory(getConf()));
-
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@@ -118,8 +114,17 @@ public class DefaultContainerExecutor extends ContainerExecutor {
Path tokenDst = new Path(appStorageDir, tokenFn);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
- lfs.setWorkingDirectory(appStorageDir);
- LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
+
+
+ FileContext localizerFc = FileContext.getFileContext(
+ lfs.getDefaultFileSystem(), getConf());
+ localizerFc.setUMask(lfs.getUMask());
+ localizerFc.setWorkingDirectory(appStorageDir);
+ LOG.info("Localizer CWD set to " + appStorageDir + " = "
+ + localizerFc.getWorkingDirectory());
+ ContainerLocalizer localizer =
+ new ContainerLocalizer(localizerFc, user, appId, locId,
+ getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.runLocalization(nmAddr);
}