You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/08/09 18:13:27 UTC
hadoop git commit: YARN-6804. Allow custom hostname for docker
containers in native services. Contributed by Billie Rinaldi
Repository: hadoop
Updated Branches:
refs/heads/branch-2 74bcc8d1f -> 8b7590057
YARN-6804. Allow custom hostname for docker containers in native services. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b759005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b759005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b759005
Branch: refs/heads/branch-2
Commit: 8b7590057e6a99a97522b1d995393808f2638dfc
Parents: 74bcc8d
Author: Jian He <ji...@apache.org>
Authored: Mon Jul 24 21:08:10 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Aug 9 11:12:38 2017 -0700
----------------------------------------------------------------------
.../client/binding/RegistryPathUtils.java | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 4 ++
.../runtime/DockerLinuxContainerRuntime.java | 67 +++++++++++++++-----
.../linux/runtime/docker/DockerRunCommand.java | 6 ++
.../impl/container-executor.c | 4 ++
.../test/test-container-executor.c | 16 ++---
.../runtime/TestDockerContainerRuntime.java | 58 +++++++++++++----
7 files changed, 117 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
index 5d8ea3f..5fa45f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
@@ -213,6 +213,6 @@ public class RegistryPathUtils {
* @return a string suitable for use in registry paths.
*/
public static String encodeYarnID(String yarnId) {
- return yarnId.replace("_", "-");
+ return yarnId.replace("container", "ctr").replace("_", "-");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 83b217a..13a336e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -53,6 +53,10 @@
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ </dependency>
+ <dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 6bdaf30..323466d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
@@ -101,6 +102,11 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* property.
* </li>
* <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
+ * hostname to be used by the Docker container. If not specified, a
+ * hostname will be derived from the container ID.
+ * </li>
+ * <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
* controls whether the Docker container is a privileged container. In order
* to use privileged containers, the
@@ -134,6 +140,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
"^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
private static final Pattern dockerImagePattern =
Pattern.compile(DOCKER_IMAGE_PATTERN);
+ public static final String HOSTNAME_PATTERN =
+ "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
+ private static final Pattern hostnamePattern = Pattern.compile(
+ HOSTNAME_PATTERN);
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_IMAGE =
@@ -147,6 +157,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_NETWORK =
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
+ @InterfaceAudience.Private
+ public static final String ENV_DOCKER_CONTAINER_HOSTNAME =
+ "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME";
+ @InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
@InterfaceAudience.Private
@@ -211,9 +225,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
this.privilegedOperationExecutor = privilegedOperationExecutor;
if (cGroupsHandler == null) {
- if (LOG.isInfoEnabled()) {
- LOG.info("cGroupsHandler is null - cgroups not in use.");
- }
+ LOG.info("cGroupsHandler is null - cgroups not in use.");
} else {
this.cGroupsHandler = cGroupsHandler;
}
@@ -267,6 +279,29 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
throw new ContainerExecutionException(msg);
}
+ public static void validateHostname(String hostname) throws
+ ContainerExecutionException {
+ if (hostname != null && !hostname.isEmpty()) {
+ if (!hostnamePattern.matcher(hostname).matches()) {
+ throw new ContainerExecutionException("Hostname '" + hostname
+ + "' doesn't match docker hostname pattern");
+ }
+ }
+ }
+
+ /** Set a DNS friendly hostname. */
+ private void setHostname(DockerRunCommand runCommand, String
+ containerIdStr, String name)
+ throws ContainerExecutionException {
+ if (name == null || name.isEmpty()) {
+ name = RegistryPathUtils.encodeYarnID(containerIdStr);
+ validateHostname(name);
+ }
+
+ LOG.info("setting hostname in container to: " + name);
+ runCommand.setHostname(name);
+ }
+
/**
* If CGROUPS in enabled and not set to none, then set the CGROUP parent for
* the command instance.
@@ -343,10 +378,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return false;
}
- if (LOG.isInfoEnabled()) {
- LOG.info("Privileged container requested for : " + container
- .getContainerId().toString());
- }
+ LOG.info("Privileged container requested for : " + container
+ .getContainerId().toString());
//Ok, so we have been asked to run a privileged container. Security
// checks need to be run. Each violation is an error.
@@ -375,10 +408,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
throw new ContainerExecutionException(message);
}
- if (LOG.isInfoEnabled()) {
- LOG.info("All checks pass. Launching privileged container for : "
- + container.getContainerId().toString());
- }
+ LOG.info("All checks pass. Launching privileged container for : "
+ + container.getContainerId().toString());
return true;
}
@@ -413,6 +444,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.getEnvironment();
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
+ String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
if(network == null || network.isEmpty()) {
network = defaultNetwork;
@@ -420,6 +452,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
validateContainerNetworkType(network);
+ validateHostname(hostname);
+
validateImageName(imageName);
String containerIdStr = container.getContainerId().toString();
@@ -454,12 +488,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runAsUser, imageName)
.detachOnRun()
.setContainerWorkDir(containerWorkDir.toString())
- .setNetworkType(network)
- .setCapabilities(capabilities)
+ .setNetworkType(network);
+ setHostname(runCommand, containerIdStr, hostname);
+ runCommand.setCapabilities(capabilities)
.addMountLocation(CGROUPS_ROOT_DIRECTORY,
CGROUPS_ROOT_DIRECTORY + ":ro", false);
- List<String> allDirs = new ArrayList<>(containerLocalDirs);
+ List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
allDirs.add(containerWorkDir.toString());
allDirs.addAll(containerLogDirs);
@@ -500,9 +535,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
if (disableOverride != null && disableOverride.equals("true")) {
- if (LOG.isInfoEnabled()) {
- LOG.info("command override disabled");
- }
+ LOG.info("command override disabled");
} else {
List<String> overrideCommands = new ArrayList<>();
Path launchDst =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index f79f4ed..b645754 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -91,6 +91,12 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+
+ public DockerRunCommand setHostname(String hostname) {
+ super.addCommandArguments("--hostname=" + hostname);
+ return this;
+ }
+
public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) {
super.addCommandArguments("--device=" + sourceDevice + ":" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index b3dd24b..bfafd31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1211,6 +1211,7 @@ char* sanitize_docker_command(const char *line) {
{"rm", no_argument, 0, 'r' },
{"workdir", required_argument, 0, 'w' },
{"net", required_argument, 0, 'e' },
+ {"hostname", required_argument, 0, 'h' },
{"cgroup-parent", required_argument, 0, 'g' },
{"privileged", no_argument, 0, 'p' },
{"cap-add", required_argument, 0, 'a' },
@@ -1252,6 +1253,9 @@ char* sanitize_docker_command(const char *line) {
case 'e':
quote_and_append_arg(&output, &output_size, "--net=", optarg);
break;
+ case 'h':
+ quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
+ break;
case 'v':
quote_and_append_arg(&output, &output_size, "-v ", optarg);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index c0afce3..242dade 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -1100,17 +1100,17 @@ void test_recursive_unlink_children() {
void test_sanitize_docker_command() {
char *input[] = {
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
"run ''''''''"
};
char *expected_output[] = {
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
+ "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
"run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b759005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index f611843..9894dcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -69,6 +70,7 @@ public class TestDockerContainerRuntime {
private PrivilegedOperationExecutor mockExecutor;
private CGroupsHandler mockCGroupsHandler;
private String containerId;
+ private String defaultHostname;
private Container container;
private ContainerId cId;
private ContainerLaunchContext context;
@@ -108,6 +110,7 @@ public class TestDockerContainerRuntime {
.mock(PrivilegedOperationExecutor.class);
mockCGroupsHandler = Mockito.mock(CGroupsHandler.class);
containerId = "container_id";
+ defaultHostname = RegistryPathUtils.encodeYarnID(containerId);
container = mock(Container.class);
cId = mock(ContainerId.class);
context = mock(ContainerLaunchContext.class);
@@ -287,6 +290,7 @@ public class TestDockerContainerRuntime {
.append("--user=%2$s -d ")
.append("--workdir=%3$s ")
.append("--net=host ")
+ .append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ")
@@ -365,7 +369,7 @@ public class TestDockerContainerRuntime {
String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
try {
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
disallowedNetwork);
runtime.launchContainer(builder.build());
Assert.fail("Network was expected to be disallowed: " +
@@ -378,8 +382,11 @@ public class TestDockerContainerRuntime {
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
String allowedNetwork = YarnConfiguration
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
allowedNetwork);
+ String expectedHostname = "test.hostname";
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME,
+ expectedHostname);
//this should cause no failures.
@@ -393,6 +400,7 @@ public class TestDockerContainerRuntime {
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ")
+ .append("--hostname=" + expectedHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@@ -448,6 +456,7 @@ public class TestDockerContainerRuntime {
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ")
+ .append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@@ -471,7 +480,7 @@ public class TestDockerContainerRuntime {
//now set an explicit (non-default) allowedNetwork and ensure that it is
// used.
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
customNetwork2);
runtime.launchContainer(builder.build());
@@ -485,6 +494,7 @@ public class TestDockerContainerRuntime {
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ")
+ .append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@@ -505,7 +515,7 @@ public class TestDockerContainerRuntime {
//disallowed network should trigger a launch failure
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
+ env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK,
customNetwork3);
try {
runtime.launchContainer(builder.build());
@@ -524,8 +534,8 @@ public class TestDockerContainerRuntime {
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
- "invalid-value");
+ env.put(DockerLinuxContainerRuntime
+ .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value");
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@@ -552,8 +562,8 @@ public class TestDockerContainerRuntime {
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
- "true");
+ env.put(DockerLinuxContainerRuntime
+ .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
try {
runtime.launchContainer(builder.build());
@@ -575,8 +585,8 @@ public class TestDockerContainerRuntime {
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
- "true");
+ env.put(DockerLinuxContainerRuntime
+ .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
//By default
// yarn.nodemanager.runtime.linux.docker.privileged-containers.acl
// is empty. So we expect this launch to fail.
@@ -605,8 +615,8 @@ public class TestDockerContainerRuntime {
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
- "true");
+ env.put(DockerLinuxContainerRuntime
+ .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
try {
runtime.launchContainer(builder.build());
@@ -632,8 +642,8 @@ public class TestDockerContainerRuntime {
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf);
- env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
- "true");
+ env.put(DockerLinuxContainerRuntime
+ .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true");
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@@ -927,4 +937,24 @@ public class TestDockerContainerRuntime {
}
}
}
+
+ @Test
+ public void testDockerHostnamePattern() throws Exception {
+ String[] validNames = {"ab", "a.b.c.d", "a1-b.cd.ef", "0AB.", "C_D-"};
+
+ String[] invalidNames = {"a", "a#.b.c", "-a.b.c", "a@b.c", "a/b/c"};
+
+ for (String name : validNames) {
+ DockerLinuxContainerRuntime.validateHostname(name);
+ }
+
+ for (String name : invalidNames) {
+ try {
+ DockerLinuxContainerRuntime.validateHostname(name);
+ Assert.fail(name + " is an invalid hostname and should fail the regex");
+ } catch (ContainerExecutionException ce) {
+ continue;
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org