You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/09/23 22:32:58 UTC
[06/21] hadoop git commit: YARN-4095. Avoid sharing
AllocatorPerContext object in LocalDirAllocator between ShuffleHandler and
LocalDirsHandlerService. Contributed by Zhihai Xu
YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator between ShuffleHandler and LocalDirsHandlerService. Contributed by Zhihai Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c890c51a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c890c51a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c890c51a
Branch: refs/heads/YARN-1197
Commit: c890c51a916894a985439497b8a44e8eee82d762
Parents: a2c76e5
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Sep 23 15:42:01 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Sep 23 15:42:01 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../nodemanager/LocalDirsHandlerService.java | 33 +++++++++++++++++---
.../TestLocalDirsHandlerService.java | 18 +++++++++++
3 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c890c51a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 329e743..3cd92f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -460,6 +460,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12428. Fix inconsistency between log-level guards and statements.
(Jagadesh Kiran N and Jackie Chang via ozawa)
+ YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator
+ between ShuffleHandler and LocalDirsHandlerService. (Zhihai Xu via jlowe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c890c51a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 6709c90..769044a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -30,6 +30,7 @@ import java.util.TimerTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -52,6 +53,22 @@ public class LocalDirsHandlerService extends AbstractService {
private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class);
+ /**
+ * Good local directories, use internally,
+ * initial value is the same as NM_LOCAL_DIRS.
+ */
+ @Private
+ static final String NM_GOOD_LOCAL_DIRS =
+ YarnConfiguration.NM_PREFIX + "good-local-dirs";
+
+ /**
+ * Good log directories, use internally,
+ * initial value is the same as NM_LOG_DIRS.
+ */
+ @Private
+ static final String NM_GOOD_LOG_DIRS =
+ YarnConfiguration.NM_PREFIX + "good-log-dirs";
+
/** Timer used to schedule disk health monitoring code execution */
private Timer dirsHandlerScheduler;
private long diskHealthCheckInterval;
@@ -113,9 +130,17 @@ public class LocalDirsHandlerService extends AbstractService {
new DirectoryCollection(
validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)),
maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB);
+
+ String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS);
+ conf.set(NM_GOOD_LOCAL_DIRS,
+ (local != null) ? local : "");
localDirsAllocator = new LocalDirAllocator(
- YarnConfiguration.NM_LOCAL_DIRS);
- logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
+ NM_GOOD_LOCAL_DIRS);
+ String log = conf.get(YarnConfiguration.NM_LOG_DIRS);
+ conf.set(NM_GOOD_LOG_DIRS,
+ (log != null) ? log : "");
+ logDirsAllocator = new LocalDirAllocator(
+ NM_GOOD_LOG_DIRS);
}
@Override
@@ -373,10 +398,10 @@ public class LocalDirsHandlerService extends AbstractService {
Configuration conf = getConfig();
List<String> localDirs = getLocalDirs();
- conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS,
+ conf.setStrings(NM_GOOD_LOCAL_DIRS,
localDirs.toArray(new String[localDirs.size()]));
List<String> logDirs = getLogDirs();
- conf.setStrings(YarnConfiguration.NM_LOG_DIRS,
+ conf.setStrings(NM_GOOD_LOG_DIRS,
logDirs.toArray(new String[logDirs.size()]));
if (!areDisksHealthy()) {
// Just log.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c890c51a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
index c61d1f0..e704c8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
@@ -120,6 +120,15 @@ public class TestLocalDirsHandlerService {
Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());
+ Assert.assertEquals("",
+ dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
+ Assert.assertEquals("",
+ dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
+ Assert.assertEquals(localDir1 + "," + localDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
+ Assert.assertEquals(logDir1 + "," + logDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
+
conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
100.0f);
nm = NodeManagerMetrics.create();
@@ -141,6 +150,15 @@ public class TestLocalDirsHandlerService {
Assert
.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());
+ Assert.assertEquals(localDir2,
+ dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
+ Assert.assertEquals(logDir2,
+ dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
+ Assert.assertEquals(localDir1 + "," + localDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
+ Assert.assertEquals(logDir1 + "," + logDir2,
+ dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
+
FileUtils.deleteDirectory(new File(localDir1));
FileUtils.deleteDirectory(new File(localDir2));
FileUtils.deleteDirectory(new File(logDir1));