You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2018/03/19 22:39:46 UTC
hadoop git commit: YARN-8027. Setting hostname of docker container
breaks for --net=host in docker 1.13. Contributed by Jim Brennan
Repository: hadoop
Updated Branches:
refs/heads/trunk a08921ca6 -> f480367af
YARN-8027. Setting hostname of docker container breaks for --net=host in docker 1.13. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f480367a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f480367a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f480367a
Branch: refs/heads/trunk
Commit: f480367af68e06ed17b8018092c9987a13bb9f63
Parents: a08921c
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Mar 19 16:59:58 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Mar 19 16:59:58 2018 -0500
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 10 +-
.../runtime/TestDockerContainerRuntime.java | 98 +++++++++++++++-----
2 files changed, 81 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f480367a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index de225e6..0290493 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -133,7 +133,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
* hostname to be used by the Docker container. If not specified, a
- * hostname will be derived from the container ID.
+ * hostname will be derived from the container ID. This variable is
+ * ignored if the network is 'host' and Registry DNS is not enabled.
* </li>
* <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
@@ -785,7 +786,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.detachOnRun()
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network);
- setHostname(runCommand, containerIdStr, hostname);
+ // Only add hostname if network is not host or if Registry DNS is enabled.
+ if (!network.equalsIgnoreCase("host") ||
+ conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
+ RegistryConstants.DEFAULT_DNS_ENABLED)) {
+ setHostname(runCommand, containerIdStr, hostname);
+ }
runCommand.setCapabilities(capabilities);
runCommand.addAllReadWriteMountLocations(containerLogDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f480367a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 4c53eb1..132a202 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.Shell;
@@ -202,6 +203,8 @@ public class TestDockerContainerRuntime {
LOG.info("Could not run id -G command: " + e);
}
uidGidPair = uid + ":" + gid;
+ // Prevent gid threshold failures for these tests
+ conf.setInt(YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD, 0);
user = "user";
appId = "app_id";
@@ -340,7 +343,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -352,7 +355,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -390,7 +392,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(15, dockerCommands.size());
+ Assert.assertEquals(14, dockerCommands.size());
int counter = 0;
Assert.assertEquals("[docker-command-execution]",
dockerCommands.get(counter++));
@@ -401,8 +403,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id",
- dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -488,10 +488,7 @@ public class TestDockerContainerRuntime {
LOG.info("Caught expected exception: " + e);
}
- int size = YarnConfiguration
- .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
- String allowedNetwork = YarnConfiguration
- .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
+ String allowedNetwork = "bridge";
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
allowedNetwork);
String expectedHostname = "test.hostname";
@@ -544,6 +541,64 @@ public class TestDockerContainerRuntime {
@Test
@SuppressWarnings("unchecked")
+ public void testContainerLaunchWithHostDnsNetwork()
+ throws ContainerExecutionException, IOException,
+ PrivilegedOperationException {
+ // Make it look like Registry DNS is enabled so we can test whether
+ // hostname goes through
+ conf.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+ DockerLinuxContainerRuntime runtime =
+ new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, null);
+
+ String expectedHostname = "test.hostname";
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME,
+ expectedHostname);
+
+ runtime.launchContainer(builder.build());
+ PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ //This is the expected docker invocation for this case
+ List<String> dockerCommands = Files
+ .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+ int expected = 15;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=test.hostname",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
+ + "/test_application_local_dir:/test_application_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
public void testContainerLaunchWithCustomNetworks()
throws ContainerExecutionException, IOException,
PrivilegedOperationException {
@@ -686,7 +741,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -737,7 +792,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 16;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -749,7 +804,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -789,7 +843,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -895,7 +949,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 16;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -907,7 +961,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -1012,7 +1065,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1024,7 +1077,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert.assertEquals(" image=busybox:latest",
dockerCommands.get(counter++));
Assert.assertEquals(
@@ -1085,7 +1137,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1097,7 +1149,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert.assertEquals(" image=busybox:latest",
dockerCommands.get(counter++));
Assert.assertEquals(
@@ -1140,7 +1191,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1152,7 +1203,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert.assertEquals(" image=busybox:latest",
dockerCommands.get(counter++));
Assert.assertEquals(
@@ -1635,7 +1685,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 16;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1647,7 +1697,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
Assert.assertEquals(
@@ -1775,7 +1824,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 16;
+ int expected = 15;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1789,7 +1838,6 @@ public class TestDockerContainerRuntime {
dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
- Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
Assert.assertEquals(" image=busybox:latest",
dockerCommands.get(counter++));
Assert.assertEquals(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org