You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/11/04 01:31:24 UTC
[19/50] [abbrv] hadoop git commit: YARN-7286. Add support for docker
to have no capabilities. Contributed by Eric Badger
YARN-7286. Add support for docker to have no capabilities. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d00b6f7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d00b6f7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d00b6f7c
Branch: refs/heads/HDFS-9806
Commit: d00b6f7c1ff2d7569ae9efdc6823ebcfb86ef2d4
Parents: cc3f3ec
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Nov 2 09:37:17 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Nov 2 09:37:17 2017 -0500
----------------------------------------------------------------------
.../src/main/resources/yarn-default.xml | 3 +-
.../runtime/DockerLinuxContainerRuntime.java | 30 ++++++++++++--
.../runtime/TestDockerContainerRuntime.java | 43 ++++++++++++++++++++
3 files changed, 71 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8487e72..f4b2e61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1623,7 +1623,8 @@
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
- uppercase.</description>
+ uppercase. To run without any capabilites, set this value to
+ "none" or "NONE"</description>
<name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
<value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 6f7b6fd..a425cf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -58,6 +58,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -187,6 +188,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private boolean enableUserReMapping;
private int userRemappingUidThreshold;
private int userRemappingGidThreshold;
+ private Set<String> capabilities;
/**
* Return whether the given environment variables indicate that the operation
@@ -285,6 +287,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
userRemappingGidThreshold = conf.getInt(
YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
+
+ capabilities = getDockerCapabilitiesFromConf();
+ }
+
+ private Set<String> getDockerCapabilitiesFromConf() throws
+ ContainerExecutionException {
+ Set<String> caps = new HashSet<>(Arrays.asList(
+ conf.getTrimmedStrings(
+ YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+ if(caps.contains("none") || caps.contains("NONE")) {
+ if(caps.size() > 1) {
+ String msg = "Mixing capabilities with the none keyword is" +
+ " not supported";
+ throw new ContainerExecutionException(msg);
+ }
+ caps = Collections.emptySet();
+ }
+
+ return caps;
+ }
+
+ public Set<String> getCapabilities() {
+ return capabilities;
}
@Override
@@ -602,10 +628,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
- Set<String> capabilities = new HashSet<>(Arrays.asList(
- conf.getTrimmedStrings(
- YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
- YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@SuppressWarnings("unchecked")
DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d00b6f7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index b5a6497..76aca04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -65,6 +65,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -1304,4 +1305,46 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
}
+
+ @Test
+ public void testDockerCapabilities()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ try {
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "none", "CHOWN", "DAC_OVERRIDE");
+ runtime.initialize(conf, null);
+ Assert.fail("Initialize didn't fail with invalid capabilities " +
+ "'none', 'CHOWN', 'DAC_OVERRIDE'");
+ } catch (ContainerExecutionException e) {
+ }
+
+ try {
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "CHOWN", "DAC_OVERRIDE", "NONE");
+ runtime.initialize(conf, null);
+ Assert.fail("Initialize didn't fail with invalid capabilities " +
+ "'CHOWN', 'DAC_OVERRIDE', 'NONE'");
+ } catch (ContainerExecutionException e) {
+ }
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "NONE");
+ runtime.initialize(conf, null);
+ Assert.assertEquals(0, runtime.getCapabilities().size());
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "none");
+ runtime.initialize(conf, null);
+ Assert.assertEquals(0, runtime.getCapabilities().size());
+
+ conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+ "CHOWN", "DAC_OVERRIDE");
+ runtime.initialize(conf, null);
+ Iterator<String> it = runtime.getCapabilities().iterator();
+ Assert.assertEquals("CHOWN", it.next());
+ Assert.assertEquals("DAC_OVERRIDE", it.next());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org