You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2018/09/27 19:45:03 UTC
hadoop git commit: YARN-6456. Added config to set default container
runtimes. Contributed by Craig Condit
Repository: hadoop
Updated Branches:
refs/heads/trunk 184544eff -> b237a0dd4
YARN-6456. Added config to set default container runtimes.
Contributed by Craig Condit
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b237a0dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b237a0dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b237a0dd
Branch: refs/heads/trunk
Commit: b237a0dd44ab285941983648d7ef26b99b30d624
Parents: 184544e
Author: Eric Yang <ey...@apache.org>
Authored: Thu Sep 27 15:28:38 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Sep 27 15:31:18 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 ++
.../src/main/resources/yarn-default.xml | 13 +++
.../container/ContainerImpl.java | 3 +
.../launcher/ContainerLaunch.java | 1 +
.../resources/gpu/GpuResourceHandlerImpl.java | 3 +
.../runtime/DefaultLinuxContainerRuntime.java | 6 +-
.../runtime/DockerLinuxContainerRuntime.java | 23 +++--
.../resources/gpu/TestGpuResourceHandler.java | 9 +-
.../runtime/TestDockerContainerRuntime.java | 91 +++++++++++++++++++-
.../src/site/markdown/DockerContainers.md | 17 ++++
10 files changed, 159 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d69ae57..95861d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1882,9 +1882,17 @@ public class YarnConfiguration extends Configuration {
public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
= {"default"};
+ /** Default runtime to be used. */
+ public static final String LINUX_CONTAINER_RUNTIME_TYPE =
+ LINUX_CONTAINER_RUNTIME_PREFIX + "type";
+
public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
+ /** Default docker image to be used. */
+ public static final String NM_DOCKER_IMAGE_NAME =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
+
/** Capabilities allowed (and added by default) for docker containers. **/
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0700902..e6f7b37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1726,6 +1726,12 @@
</property>
<property>
+ <description>Default container runtime to use.</description>
+ <name>yarn.nodemanager.runtime.linux.type</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
@@ -1736,6 +1742,13 @@
</property>
<property>
+ <description>Default docker image to be used when the docker runtime is
+ selected.</description>
+ <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+ <value></value>
+ </property>
+
+ <property>
<description>This configuration setting determines if privileged docker
containers are allowed on this cluster. Privileged containers are granted
the complete set of capabilities and are not subject to the limitations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index e4cbfdc..f88dfbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1544,6 +1544,7 @@ public class ContainerImpl implements Container {
// TODO: Add containerWorkDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@@ -1584,6 +1585,7 @@ public class ContainerImpl implements Container {
// TODO: Add containerOuputDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@@ -1858,6 +1860,7 @@ public class ContainerImpl implements Container {
}
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6347d4e..3fa73ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -799,6 +799,7 @@ public class ContainerLaunch implements Callable<Integer> {
}
// The Docker container may not have fully started, reap the container.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+ conf,
container.getLaunchContext().getEnvironment())) {
reapDockerContainerNoPid(user);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 1184382..e25a6b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
+ private Context nmContext;
private GpuResourceAllocator gpuAllocator;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public GpuResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor) {
+ this.nmContext = nmContext;
this.cGroupsHandler = cGroupsHandler;
this.privilegedOperationExecutor = privilegedOperationExecutor;
gpuAllocator = new GpuResourceAllocator(nmContext);
@@ -102,6 +104,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
containerIdStr);
if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
+ nmContext.getConf(),
container.getLaunchContext().getEnvironment())) {
// Write to devices cgroup only for non-docker container. The reason is
// docker engine runtime runc do the devices cgroups initialize in the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 2b8e8ad..82ca6d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -71,7 +72,10 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
- return type == null || type.equals("default");
+ if (type == null) {
+ type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
+ }
+ return type == null || type.isEmpty() || type.equals("default");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 55dfdc3..c86ed53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private Context nmContext;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
+ private String defaultImageName;
private Set<String> allowedNetworks = new HashSet<>();
private String defaultNetwork;
private CGroupsHandler cGroupsHandler;
@@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
* called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
* this method will return true. Otherwise it will return false.
*
+ * @param daemonConf the NodeManager daemon configuration
* @param env the environment variable settings for the operation
* @return whether a Docker container is requested
*/
- public static boolean isDockerContainerRequested(
+ public static boolean isDockerContainerRequested(Configuration daemonConf,
Map<String, String> env) {
- if (env == null) {
- return false;
+ String type = (env == null)
+ ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+ if (type == null) {
+ type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
-
- String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
-
return type != null && type.equals("docker");
}
@@ -312,6 +313,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
defaultROMounts.clear();
defaultRWMounts.clear();
defaultTmpfsMounts.clear();
+ defaultImageName = conf.getTrimmed(
+ YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
allowedNetworks.addAll(Arrays.asList(
conf.getTrimmedStrings(
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@@ -325,8 +328,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+ " is not in the set of allowed networks: " + allowedNetworks;
if (LOG.isWarnEnabled()) {
- LOG.warn(message + ". Please check "
- + "configuration");
+ LOG.warn(message + ". Please check configuration");
}
throw new ContainerExecutionException(message);
@@ -369,7 +371,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
- return isDockerContainerRequested(env);
+ return isDockerContainerRequested(conf, env);
}
private Set<String> getDockerCapabilitiesFromConf() throws
@@ -789,6 +791,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
boolean useEntryPoint = checkUseEntryPoint(environment);
+ if (imageName == null || imageName.isEmpty()) {
+ imageName = defaultImageName;
+ }
if(network == null || network.isEmpty()) {
network = defaultNetwork;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 9a8a4c9..301c3e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -80,8 +80,11 @@ public class TestGpuResourceHandler {
mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
mockNMStateStore = mock(NMStateStoreService.class);
+ Configuration conf = new Configuration();
+
Context nmctx = mock(Context.class);
when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
+ when(nmctx.getConf()).thenReturn(conf);
runningContainersMap = new ConcurrentHashMap<>();
when(nmctx.getContainers()).thenReturn(runningContainersMap);
@@ -347,15 +350,17 @@ public class TestGpuResourceHandler {
public void testAllocationStoredWithNULLStateStore() throws Exception {
NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
+
Context nmnctx = mock(Context.class);
when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
+ when(nmnctx.getConf()).thenReturn(conf);
GpuResourceHandlerImpl gpuNULLStateResourceHandler =
new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
mockPrivilegedExecutor);
- Configuration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuNULLStateResourceHandler.bootstrap(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 4464496..a880ba9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -309,11 +309,45 @@ public class TestDockerContainerRuntime {
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
Assert.assertEquals(false, DockerLinuxContainerRuntime
- .isDockerContainerRequested(null));
+ .isDockerContainerRequested(conf, null));
Assert.assertEquals(true, DockerLinuxContainerRuntime
- .isDockerContainerRequested(envDockerType));
+ .isDockerContainerRequested(conf, envDockerType));
Assert.assertEquals(false, DockerLinuxContainerRuntime
- .isDockerContainerRequested(envOtherType));
+ .isDockerContainerRequested(conf, envOtherType));
+ }
+
+ @Test
+ public void testSelectDockerContainerTypeWithDockerAsDefault() {
+ Map<String, String> envDockerType = new HashMap<>();
+ Map<String, String> envOtherType = new HashMap<>();
+
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker");
+ envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, null));
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envDockerType));
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envOtherType));
+ }
+
+ @Test
+ public void testSelectDockerContainerTypeWithDefaultSet() {
+ Map<String, String> envDockerType = new HashMap<>();
+ Map<String, String> envOtherType = new HashMap<>();
+
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default");
+ envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, null));
+ Assert.assertEquals(true, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envDockerType));
+ Assert.assertEquals(false, DockerLinuxContainerRuntime
+ .isDockerContainerRequested(conf, envOtherType));
}
private PrivilegedOperation capturePrivilegedOperation()
@@ -422,6 +456,57 @@ public class TestDockerContainerRuntime {
}
@Test
+ public void testDockerContainerLaunchWithDefaultImage()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3");
+ env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE);
+
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, nmContext);
+ runtime.launchContainer(builder.build());
+
+ PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ List<String> dockerCommands = Files.readAllLines(Paths.get(
+ dockerCommandFile), Charset.forName("UTF-8"));
+
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " name=container_e11_1518975676334_14532816_01_000001",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter));
+ }
+
+ @Test
public void testContainerLaunchWithUserRemapping()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b237a0dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 15105f3..17a335e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -115,6 +115,23 @@ The following properties should be set in yarn-site.xml:
</property>
<property>
+ <name>yarn.nodemanager.runtime.linux.type</name>
+ <value></value>
+ <description>
+ Optional. Sets the default container runtime to use.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+ <value></value>
+ <description>
+ Optional. Default docker image to be used when the docker runtime is
+ selected.
+ </description>
+ </property>
+
+ <property>
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
<value>host,none,bridge</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org