You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/04/24 18:39:25 UTC
hadoop git commit: YARN-6500. Do not mount inaccessible cgroups
directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/trunk 52adf7191 -> 8ac50e132
YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ac50e13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ac50e13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ac50e13
Branch: refs/heads/trunk
Commit: 8ac50e1322cb3f84bd998635924d85846aa47c94
Parents: 52adf71
Author: Haibo Chen <ha...@cloudera.com>
Authored: Mon Apr 24 11:37:52 2017 -0700
Committer: Haibo Chen <ha...@cloudera.com>
Committed: Mon Apr 24 11:37:52 2017 -0700
----------------------------------------------------------------------
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../util/CgroupsLCEResourcesHandler.java | 13 ++++++--
.../linux/resources/TestCGroupsHandlerImpl.java | 32 ++++++++++++++++++++
.../util/TestCgroupsLCEResourcesHandler.java | 29 ++++++++++++++++++
4 files changed, 73 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index d5295c5..0f4c17e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @param entries map of paths to mount options
* @return the first mount path that has the requested subsystem
*/
- private static String findControllerInMtab(String controller,
+ @VisibleForTesting
+ static String findControllerInMtab(String controller,
Map<String, List<String>> entries) {
for (Map.Entry<String, List<String>> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index f04fcd2..cb4dcf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
return ret;
}
- private String findControllerInMtab(String controller,
+ @VisibleForTesting
+ String findControllerInMtab(String controller,
Map<String, List<String>> entries) {
for (Entry<String, List<String>> e : entries.entrySet()) {
- if (e.getValue().contains(controller))
- return e.getKey();
+ if (e.getValue().contains(controller)) {
+ if (new File(e.getKey()).canRead()) {
+ return e.getKey();
+ } else {
+ LOG.warn(String.format(
+ "Skipping inaccessible cgroup mount point %s", e.getKey()));
+ }
+ }
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index 4c0829e..804f8e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -40,6 +40,9 @@ import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.security.Permission;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -432,6 +435,35 @@ public class TestCGroupsHandlerImpl {
}
}
+ @Test
+ public void testSelectCgroup() throws Exception {
+ File cpu = new File(tmpPath, "cpu");
+ File cpuNoExist = new File(tmpPath, "cpuNoExist");
+ File memory = new File(tmpPath, "memory");
+ try {
+ CGroupsHandlerImpl handler = new CGroupsHandlerImpl(
+ conf,
+ privilegedOperationExecutorMock);
+ Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+ Assert.assertTrue("temp dir should be created", memory.mkdirs());
+ Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+ cgroups.put(
+ memory.getAbsolutePath(), Collections.singletonList("memory"));
+ cgroups.put(
+ cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+ cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+ String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+ Assert.assertEquals("Wrong CPU mount point selected",
+ cpu.getAbsolutePath(), selectedCPU);
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ FileUtils.deleteQuietly(memory);
+ }
+ }
+
@After
public void teardown() {
FileUtil.fullyDelete(new File(tmpPath));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 34c9ad1..83c8d5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -33,7 +33,10 @@ import org.junit.Before;
import org.mockito.Mockito;
import java.io.*;
+import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Scanner;
import java.util.concurrent.CountDownLatch;
@@ -342,4 +345,30 @@ public class TestCgroupsLCEResourcesHandler {
FileUtils.deleteQuietly(cgroupDir);
}
+ @Test
+ public void testSelectCgroup() {
+ File cpu = new File(cgroupDir, "cpu");
+ File cpuNoExist = new File(cgroupDir, "cpuNoExist");
+ File memory = new File(cgroupDir, "memory");
+ try {
+ CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+ Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+ Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+ Assert.assertTrue("temp dir should be created", memory.mkdirs());
+ Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+ cgroups.put(
+ memory.getAbsolutePath(), Collections.singletonList("memory"));
+ cgroups.put(
+ cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+ cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+ String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+ Assert.assertEquals("Wrong CPU mount point selected",
+ cpu.getAbsolutePath(), selectedCPU);
+ } finally {
+ FileUtils.deleteQuietly(cpu);
+ FileUtils.deleteQuietly(memory);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org