You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/11 06:48:48 UTC
[23/50] [abbrv] hadoop git commit: YARN-6930. Admins should be able
to explicitly enable specific LinuxContainerRuntime in the NodeManager.
Contributed by Shane Kumpf
YARN-6930. Admins should be able to explicitly enable specific LinuxContainerRuntime in the NodeManager. Contributed by Shane Kumpf
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0b535d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0b535d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0b535d9
Branch: refs/heads/YARN-5972
Commit: b0b535d9d5727cd84fd6368c6d1b38363616504e
Parents: f155ab7
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 7 16:12:09 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 7 16:17:03 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 17 +++
.../src/main/resources/yarn-default.xml | 8 ++
.../server/nodemanager/ContainerExecutor.java | 4 +-
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../DelegatingLinuxContainerRuntime.java | 79 ++++++++---
.../runtime/LinuxContainerRuntimeConstants.java | 9 ++
.../monitor/ContainersMonitorImpl.java | 4 +-
.../runtime/ContainerRuntime.java | 6 +-
.../TestDelegatingLinuxContainerRuntime.java | 137 ++++++++++++++++++
.../src/site/markdown/DockerContainers.md | 139 ++++++++++---------
10 files changed, 320 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 27ca957..be63233 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1470,6 +1470,23 @@ public class YarnConfiguration extends Configuration {
/** Prefix for runtime configuration constants. */
public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
"runtime.linux.";
+
+ /**
+ * Comma separated list of runtimes that are allowed when using
+ * LinuxContainerExecutor. The allowed values are:
+ * <ul>
+ * <li>default</li>
+ * <li>docker</li>
+ * <li>javasandbox</li>
+ * </ul>
+ */
+ public static final String LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES =
+ LINUX_CONTAINER_RUNTIME_PREFIX + "allowed-runtimes";
+
+ /** The default list of allowed runtimes when using LinuxContainerExecutor. */
+ public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
+ = {"default"};
+
public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0cad167..afde222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1572,6 +1572,14 @@
</property>
<property>
+ <description>Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The allowed values are default, docker, and
+ javasandbox.</description>
+ <name>yarn.nodemanager.runtime.linux.allowed-runtimes</name>
+ <value>default</value>
+ </property>
+
+ <property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
be case-sensitive from a docker perspective, it is best to keep these
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index b6fb4ec..072cca7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions.ConfigurationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
@@ -663,7 +664,8 @@ public abstract class ContainerExecutor implements Configurable {
}
// LinuxContainerExecutor overrides this method and behaves differently.
- public String[] getIpAndHost(Container container) {
+ public String[] getIpAndHost(Container container)
+ throws ContainerExecutionException {
return getLocalIpAndHost(container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index dc68680..2971f83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -625,7 +625,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
- public String[] getIpAndHost(Container container) {
+ public String[] getIpAndHost(Container container)
+ throws ContainerExecutionException {
return linuxContainerRuntime.getIpAndHost(container);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 5273334..9fe4927 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -20,9 +20,11 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
@@ -31,6 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.Contai
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.EnumSet;
import java.util.Map;
/**
@@ -50,34 +53,62 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
private JavaSandboxLinuxContainerRuntime javaSandboxLinuxContainerRuntime;
+ private EnumSet<LinuxContainerRuntimeConstants.RuntimeType> allowedRuntimes =
+ EnumSet.noneOf(LinuxContainerRuntimeConstants.RuntimeType.class);
@Override
public void initialize(Configuration conf)
throws ContainerExecutionException {
- PrivilegedOperationExecutor privilegedOperationExecutor =
- PrivilegedOperationExecutor.getInstance(conf);
- defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime(
- privilegedOperationExecutor);
- defaultLinuxContainerRuntime.initialize(conf);
- dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
- privilegedOperationExecutor);
- dockerLinuxContainerRuntime.initialize(conf);
- javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime(
- privilegedOperationExecutor);
- javaSandboxLinuxContainerRuntime.initialize(conf);
+ String[] configuredRuntimes = conf.getTrimmedStrings(
+ YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ YarnConfiguration.DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES);
+ for (String configuredRuntime : configuredRuntimes) {
+ try {
+ allowedRuntimes.add(
+ LinuxContainerRuntimeConstants.RuntimeType.valueOf(
+ configuredRuntime.toUpperCase()));
+ } catch (IllegalArgumentException e) {
+ throw new ContainerExecutionException("Invalid runtime set in "
+ + YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES + " : "
+ + configuredRuntime);
+ }
+ }
+ if (isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX)) {
+ javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime(
+ PrivilegedOperationExecutor.getInstance(conf));
+ javaSandboxLinuxContainerRuntime.initialize(conf);
+ }
+ if (isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER)) {
+ dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
+ PrivilegedOperationExecutor.getInstance(conf));
+ dockerLinuxContainerRuntime.initialize(conf);
+ }
+ if (isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT)) {
+ defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime(
+ PrivilegedOperationExecutor.getInstance(conf));
+ defaultLinuxContainerRuntime.initialize(conf);
+ }
}
- private LinuxContainerRuntime pickContainerRuntime(
- Map<String, String> environment){
+ @VisibleForTesting
+ LinuxContainerRuntime pickContainerRuntime(
+ Map<String, String> environment) throws ContainerExecutionException {
LinuxContainerRuntime runtime;
//Sandbox checked first to ensure DockerRuntime doesn't circumvent controls
- if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){
- runtime = javaSandboxLinuxContainerRuntime;
- } else if (DockerLinuxContainerRuntime
- .isDockerContainerRequested(environment)){
+ if (javaSandboxLinuxContainerRuntime != null &&
+ javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()){
+ runtime = javaSandboxLinuxContainerRuntime;
+ } else if (dockerLinuxContainerRuntime != null &&
+ DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
runtime = dockerLinuxContainerRuntime;
- } else {
+ } else if (defaultLinuxContainerRuntime != null &&
+ !DockerLinuxContainerRuntime.isDockerContainerRequested(environment)) {
runtime = defaultLinuxContainerRuntime;
+ } else {
+ throw new ContainerExecutionException("Requested runtime not allowed.");
}
if (LOG.isDebugEnabled()) {
@@ -88,7 +119,8 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
return runtime;
}
- private LinuxContainerRuntime pickContainerRuntime(Container container) {
+ private LinuxContainerRuntime pickContainerRuntime(Container container)
+ throws ContainerExecutionException {
return pickContainerRuntime(container.getLaunchContext().getEnvironment());
}
@@ -127,8 +159,15 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
- public String[] getIpAndHost(Container container) {
+ public String[] getIpAndHost(Container container)
+ throws ContainerExecutionException {
LinuxContainerRuntime runtime = pickContainerRuntime(container);
return runtime.getIpAndHost(container);
}
+
+ @VisibleForTesting
+ boolean isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType runtimeType) {
+ return allowedRuntimes.contains(runtimeType);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index 2e632fa..3a47523 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -31,6 +31,15 @@ public final class LinuxContainerRuntimeConstants {
private LinuxContainerRuntimeConstants() {
}
+ /**
+ * Linux container runtime types for {@link DelegatingLinuxContainerRuntime}.
+ */
+ public enum RuntimeType {
+ DEFAULT,
+ DOCKER,
+ JAVASANDBOX;
+ }
+
public static final Attribute<Map> LOCALIZED_RESOURCES = Attribute
.attribute(Map.class, "localized_resources");
public static final Attribute<List> CONTAINER_LAUNCH_PREFIX_COMMANDS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index d764e1d..2b99cc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
@@ -502,7 +503,8 @@ public class ContainersMonitorImpl extends AbstractService implements
* @param entry process tree entry to fill in
*/
private void initializeProcessTrees(
- Entry<ContainerId, ProcessTreeInfo> entry) {
+ Entry<ContainerId, ProcessTreeInfo> entry)
+ throws ContainerExecutionException {
ContainerId containerId = entry.getKey();
ProcessTreeInfo ptInfo = entry.getValue();
String pId = ptInfo.getPID();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index b15690f..7caa0ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -77,6 +77,10 @@ public interface ContainerRuntime {
/**
* Return the host and ip of the container
+ *
+ * @param container the {@link Container}
+ * @throws ContainerExecutionException if an error occurs while getting the ip
+ * and hostname
*/
- String[] getIpAndHost(Container container);
+ String[] getIpAndHost(Container container) throws ContainerExecutionException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java
new file mode 100644
index 0000000..7f4bbc4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDelegatingLinuxContainerRuntime.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * Test container runtime delegation.
+ */
+public class TestDelegatingLinuxContainerRuntime {
+
+ private DelegatingLinuxContainerRuntime delegatingLinuxContainerRuntime;
+ private Configuration conf;
+ private Map<String, String> env = new HashMap<>();
+
+ @Before
+ public void setUp() throws Exception {
+ delegatingLinuxContainerRuntime = new DelegatingLinuxContainerRuntime();
+ conf = new Configuration();
+ env.clear();
+ }
+
+ @Test
+ public void testIsRuntimeAllowedDefault() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ YarnConfiguration.DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES[0]);
+ System.out.println(conf.get(
+ YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES));
+ delegatingLinuxContainerRuntime.initialize(conf);
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX));
+ }
+
+ @Test
+ public void testIsRuntimeAllowedDocker() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "docker");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX));
+ }
+
+ @Test
+ public void testIsRuntimeAllowedJavaSandbox() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "javasandbox");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER));
+ }
+
+ @Test
+ public void testIsRuntimeAllowedMultiple() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "docker,javasandbox");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER));
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX));
+ assertFalse(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT));
+ }
+
+ @Test
+ public void testIsRuntimeAllowedAll() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "default,docker,javasandbox");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DEFAULT));
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.DOCKER));
+ assertTrue(delegatingLinuxContainerRuntime.isRuntimeAllowed(
+ LinuxContainerRuntimeConstants.RuntimeType.JAVASANDBOX));
+ }
+
+ @Test
+ public void testJavaSandboxNotAllowedButPermissive() throws Exception {
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "default,docker");
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX, "permissive");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ ContainerRuntime runtime =
+ delegatingLinuxContainerRuntime.pickContainerRuntime(env);
+ assertTrue(runtime instanceof DefaultLinuxContainerRuntime);
+ }
+
+ @Test
+ public void testJavaSandboxNotAllowedButPermissiveDockerRequested()
+ throws Exception {
+ env.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+ conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES,
+ "default,docker");
+ conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX, "permissive");
+ delegatingLinuxContainerRuntime.initialize(conf);
+ ContainerRuntime runtime =
+ delegatingLinuxContainerRuntime.pickContainerRuntime(env);
+ assertTrue(runtime instanceof DockerLinuxContainerRuntime);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b535d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 4de0a6a..bf94169 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -71,68 +71,83 @@ request. For example:
The following properties should be set in yarn-site.xml:
```xml
-<property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
- <description>
- This is the container executor setting that ensures that all applications
- are started with the LinuxContainerExecutor.
- </description>
-</property>
-
-<property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- <description>
- The POSIX group of the NodeManager. It should match the setting in
- "container-executor.cfg". This configuration is required for validating
- the secure access of the container-executor binary.
- </description>
-</property>
-
-<property>
- <name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
- <value>false</value>
- <description>
- Whether all applications should be run as the NodeManager process' owner.
- When false, applications are launched instead as the application owner.
- </description>
-</property>
-
-<property>
- <name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
- <value>host,none,bridge</value>
- <description>
- Optional. A comma-separated set of networks allowed when launching
- containers. Valid values are determined by Docker networks available from
- `docker network ls`
- </description>
-</property>
-
-<property>
- <description>The network used when launching Docker containers when no
- network is specified in the request. This network must be one of the
- (configurable) set of allowed container networks.</description>
- <name>yarn.nodemanager.runtime.linux.docker.default-container-network</name>
- <value>host</value>
-</property>
-
-<property>
- <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed</name>
- <value>false</value>
- <description>
- Optional. Whether applications are allowed to run in privileged containers.
- </description>
-</property>
-
-<property>
- <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.acl</name>
- <value></value>
- <description>
- Optional. A comma-separated list of users who are allowed to request
- privileged contains if privileged containers are allowed.
- </description>
-</property>
+<configuration>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+ <description>
+ This is the container executor setting that ensures that all applications
+ are started with the LinuxContainerExecutor.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ <description>
+ The POSIX group of the NodeManager. It should match the setting in
+ "container-executor.cfg". This configuration is required for validating
+ the secure access of the container-executor binary.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
+ <value>false</value>
+ <description>
+ Whether all applications should be run as the NodeManager process' owner.
+ When false, applications are launched instead as the application owner.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.allowed-runtimes</name>
+ <value>default,docker</value>
+ <description>
+ Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The allowed values are default, docker, and
+ javasandbox.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
+ <value>host,none,bridge</value>
+ <description>
+ Optional. A comma-separated set of networks allowed when launching
+ containers. Valid values are determined by Docker networks available from
+ `docker network ls`
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.default-container-network</name>
+ <value>host</value>
+ <description>
+ The network used when launching Docker containers when no
+ network is specified in the request. This network must be one of the
+ (configurable) set of allowed container networks.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed</name>
+ <value>false</value>
+ <description>
+ Optional. Whether applications are allowed to run in privileged
+ containers.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.acl</name>
+ <value></value>
+ <description>
+ Optional. A comma-separated list of users who are allowed to request
+ privileged contains if privileged containers are allowed.
+ </description>
+ </property>
+</configuration>
```
In addition, a container-executer.cfg file must exist and contain settings for
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org