You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/09/01 08:43:50 UTC
[1/2] hadoop git commit: YARN-5596. Fix failing unit test in
TestDockerContainerRuntime. Contributed by Sidharta Seethana.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 776eccd87 -> 34f933065
refs/heads/trunk 6c600360c -> 08f55ccbb
YARN-5596. Fix failing unit test in TestDockerContainerRuntime. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08f55ccb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08f55ccb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08f55ccb
Branch: refs/heads/trunk
Commit: 08f55ccbb075b90a6e534f4ac0f9840f2e6890d5
Parents: 6c60036
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Sep 1 14:08:51 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Sep 1 14:08:51 2016 +0530
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 6 ++++--
.../runtime/TestDockerContainerRuntime.java | 20 ++++++++++++++++----
2 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f55ccb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 33b8add..a3aff2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -146,6 +146,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
+ static final String CGROUPS_ROOT_DIRECTORY = "/sys/fs/cgroup";
+
private Configuration conf;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -437,7 +439,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -449,7 +450,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network)
.setCapabilities(capabilities)
- .addMountLocation("/sys/fs/cgroup", "/sys/fs/cgroup:ro", false);
+ .addMountLocation(CGROUPS_ROOT_DIRECTORY,
+ CGROUPS_ROOT_DIRECTORY + ":ro", false);
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f55ccb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index a05ff46..3253394 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -257,6 +257,18 @@ public class TestDockerContainerRuntime {
return expectedCapabilitiesString.toString();
}
+ private String getExpectedCGroupsMountString() {
+ boolean cGroupsMountExists = new File(
+ DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY).exists();
+
+ if(cGroupsMountExists) {
+ return "-v " + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY
+ + ":" + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY + ":ro ";
+ } else {
+ return "";
+ }
+ }
+
@Test
public void testDockerContainerLaunch()
throws ContainerExecutionException, PrivilegedOperationException,
@@ -276,7 +288,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=host ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
@@ -382,7 +394,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -437,7 +449,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -474,7 +486,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-5596. Fix failing unit test in
TestDockerContainerRuntime. Contributed by Sidharta Seethana.
Posted by vv...@apache.org.
YARN-5596. Fix failing unit test in TestDockerContainerRuntime. Contributed by Sidharta Seethana.
(cherry picked from commit 08f55ccbb075b90a6e534f4ac0f9840f2e6890d5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34f93306
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34f93306
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34f93306
Branch: refs/heads/branch-2
Commit: 34f933065175eb0b9c8a9823d5b31e436bffd33f
Parents: 776eccd
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Sep 1 14:08:51 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Sep 1 14:10:18 2016 +0530
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 6 ++++--
.../runtime/TestDockerContainerRuntime.java | 20 ++++++++++++++++----
2 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34f93306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 33b8add..a3aff2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -146,6 +146,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
+ static final String CGROUPS_ROOT_DIRECTORY = "/sys/fs/cgroup";
+
private Configuration conf;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -437,7 +439,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -449,7 +450,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network)
.setCapabilities(capabilities)
- .addMountLocation("/sys/fs/cgroup", "/sys/fs/cgroup:ro", false);
+ .addMountLocation(CGROUPS_ROOT_DIRECTORY,
+ CGROUPS_ROOT_DIRECTORY + ":ro", false);
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34f93306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index a05ff46..3253394 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -257,6 +257,18 @@ public class TestDockerContainerRuntime {
return expectedCapabilitiesString.toString();
}
+ private String getExpectedCGroupsMountString() {
+ boolean cGroupsMountExists = new File(
+ DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY).exists();
+
+ if(cGroupsMountExists) {
+ return "-v " + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY
+ + ":" + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY + ":ro ";
+ } else {
+ return "";
+ }
+ }
+
@Test
public void testDockerContainerLaunch()
throws ContainerExecutionException, PrivilegedOperationException,
@@ -276,7 +288,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=host ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
@@ -382,7 +394,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -437,7 +449,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -474,7 +486,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org