You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2019/02/12 21:19:00 UTC
[hadoop] branch trunk updated: YARN-9184. Add a system flag to
allow update to latest docker images. Contributed by Zhaohui Xin
This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3dc2523 YARN-9184. Add a system flag to allow update to latest docker images. Contributed by Zhaohui Xin
3dc2523 is described below
commit 3dc252326693170ac1b31bf2914bae72ca73d31a
Author: Eric Yang <ey...@apache.org>
AuthorDate: Tue Feb 12 16:16:35 2019 -0500
YARN-9184. Add a system flag to allow update to latest docker images.
Contributed by Zhaohui Xin
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 +
.../src/main/resources/yarn-default.xml | 7 ++
.../linux/runtime/DockerLinuxContainerRuntime.java | 29 +++++-
.../linux/runtime/TestDockerContainerRuntime.java | 115 +++++++++++++++++++++
.../src/site/markdown/DockerContainers.md | 9 ++
5 files changed, 163 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04d0fd1..4cd4cca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1936,6 +1936,10 @@ public class YarnConfiguration extends Configuration {
public static final String NM_DOCKER_IMAGE_NAME =
DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
+ /** Default option to decide whether to pull the latest image or not. **/
+ public static final String NM_DOCKER_IMAGE_UPDATE =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "image-update";
+
/** Capabilities allowed (and added by default) for docker containers. **/
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index bfcbf4b..db29fb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1762,6 +1762,13 @@
</property>
<property>
+ <description>Default option to decide whether to pull the latest image
+ or not.</description>
+ <name>yarn.nodemanager.runtime.linux.docker.image-update</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>This configuration setting determines if privileged docker
containers are allowed on this cluster. Privileged containers are granted
the complete set of capabilities and are not subject to the limitations
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 7fc386d..f1da846 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerExecCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerKillCommand;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerPullCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStartCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerVolumeCommand;
@@ -272,6 +273,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private Map<String, CsiAdaptorProtocol> csiClients = new HashMap<>();
private PrivilegedOperationExecutor privilegedOperationExecutor;
private String defaultImageName;
+ private Boolean defaultImageUpdate;
private Set<String> allowedNetworks = new HashSet<>();
private String defaultNetwork;
private CGroupsHandler cGroupsHandler;
@@ -352,6 +354,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
defaultTmpfsMounts.clear();
defaultImageName = conf.getTrimmed(
YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
+ defaultImageUpdate = conf.getBoolean(
+ YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false);
allowedNetworks.addAll(Arrays.asList(
conf.getTrimmedStrings(
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@@ -802,6 +806,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
throws ContainerExecutionException {
Container container = ctx.getContainer();
ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
Map<String, String> environment = container.getLaunchContext()
.getEnvironment();
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
@@ -822,7 +827,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
validateImageName(imageName);
- String containerIdStr = containerId.toString();
+ if (defaultImageUpdate) {
+ pullImageFromRemote(containerIdStr, imageName);
+ }
+
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
String dockerRunAsUser = runAsUser;
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
@@ -1379,6 +1387,25 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ public void pullImageFromRemote(String containerIdStr, String imageName)
+ throws ContainerExecutionException {
+ long start = System.currentTimeMillis();
+ DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName);
+ LOG.debug("now pulling docker image." + " image name: " + imageName + ","
+ + " container: " + containerIdStr);
+
+ DockerCommandExecutor.executeDockerCommand(dockerPullCommand,
+ containerIdStr, null,
+ privilegedOperationExecutor, false, nmContext);
+
+ long end = System.currentTimeMillis();
+ long pullImageTimeMs = end - start;
+ LOG.debug("pull docker image done with "
+ + String.valueOf(pullImageTimeMs) + "ms spent."
+ + " image name: " + imageName + ","
+ + " container: " + containerIdStr);
+ }
+
private void executeLivelinessCheck(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
String procFs = ctx.getExecutionAttribute(PROCFS);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index ef0f2e0..6669cac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -534,6 +534,121 @@ public class TestDockerContainerRuntime {
}
@Test
+ public void testDockerContainerLaunchWithoutDefaultImageUpdate()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ conf.setBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false);
+
+ runtime.initialize(conf, nmContext);
+ runtime.launchContainer(builder.build());
+ List<String> dockerCommands = readDockerCommands();
+ Assert.assertEquals(false,
+ conf.getBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, false));
+
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " name=container_e11_1518975676334_14532816_01_000001",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter));
+ }
+
+ @Test
+ public void testDockerContainerLaunchWithDefaultImageUpdate()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException {
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ conf.setBoolean(YarnConfiguration.NM_DOCKER_IMAGE_UPDATE, true);
+
+ runtime.initialize(conf, nmContext);
+ runtime.launchContainer(builder.build());
+
+ ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
+ PrivilegedOperation.class);
+
+ //Two invocations expected.
+ verify(mockExecutor, times(2))
+ .executePrivilegedOperation(any(), opCaptor.capture(), any(),
+ any(), anyBoolean(), anyBoolean());
+
+ List<PrivilegedOperation> allCaptures = opCaptor.getAllValues();
+
+ // pull image from remote hub firstly
+ PrivilegedOperation op = allCaptures.get(0);
+ Assert.assertEquals(PrivilegedOperation.OperationType
+ .RUN_DOCKER_CMD, op.getOperationType());
+
+ File commandFile = new File(StringUtils.join(",", op.getArguments()));
+ FileInputStream fileInputStream = new FileInputStream(commandFile);
+ String fileContent = new String(IOUtils.toByteArray(fileInputStream));
+ Assert.assertEquals("[docker-command-execution]\n"
+ + " docker-command=pull\n"
+ + " image=busybox:latest\n", fileContent);
+ fileInputStream.close();
+
+ // launch docker container
+ List<String> dockerCommands = readDockerCommands(2);
+
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " name=container_e11_1518975676334_14532816_01_000001",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter));
+ }
+
+ @Test
public void testContainerLaunchWithUserRemapping()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 8797cb5..135a0fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -132,6 +132,15 @@ The following properties should be set in yarn-site.xml:
</property>
<property>
+ <name>yarn.nodemanager.runtime.linux.docker.image-update</name>
+ <value>false</value>
+ <description>
+ Optional. Default option to decide whether to pull the latest image
+ or not.
+ </description>
+ </property>
+
+ <property>
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
<value>host,none,bridge</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org