You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/07/25 01:32:18 UTC
[35/50] hadoop git commit: YARN-8380. Support bind propagation
options for mounts in docker runtime. Contributed by Billie Rinaldi
YARN-8380. Support bind propagation options for mounts in docker runtime.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8688a0c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8688a0c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8688a0c7
Branch: refs/heads/HADOOP-15461
Commit: 8688a0c7f88f2adf1a7fce695e06f3dd1f745080
Parents: 17e2616
Author: Eric Yang <ey...@apache.org>
Authored: Mon Jul 23 20:12:04 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Jul 23 20:12:04 2018 -0400
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 37 ++-
.../linux/runtime/docker/DockerRunCommand.java | 18 +-
.../container-executor/impl/utils/docker-util.c | 196 ++++++++------
.../test/utils/test_docker_util.cc | 133 +++++-----
.../runtime/TestDockerContainerRuntime.java | 259 +++++++++----------
.../gpu/TestNvidiaDockerV1CommandPlugin.java | 2 +-
.../src/site/markdown/DockerContainers.md | 13 +-
7 files changed, 349 insertions(+), 309 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index c89d5fb..88e6c91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -154,9 +154,13 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
+ additional volume mounts for the Docker container. The value of the
* environment variable should be a comma-separated list of mounts.
- * All such mounts must be given as {@code source:dest:mode}, and the mode
+ * All such mounts must be given as {@code source:dest[:mode]} and the mode
* must be "ro" (read-only) or "rw" (read-write) to specify the type of
- * access being requested. The requested mounts will be validated by
+ * access being requested. If neither is specified, read-write will be
+ * assumed. The mode may include a bind propagation option. In that case,
+ * the mode should either be of the form [option], rw+[option], or
+ * ro+[option]. Valid bind propagation options are shared, rshared, slave,
+ * rslave, private, and rprivate. The requested mounts will be validated by
* container-executor based on the values set in container-executor.cfg for
* {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
* </li>
@@ -189,7 +193,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private static final Pattern hostnamePattern = Pattern.compile(
HOSTNAME_PATTERN);
private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
- "(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)");
+ "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" +
+ "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)");
private static final int HOST_NAME_LENGTH = 64;
private static final String DEFAULT_PROCFS = "/proc";
@@ -840,24 +845,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
+ environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
}
parsedMounts.reset();
+ long mountCount = 0;
while (parsedMounts.find()) {
+ mountCount++;
String src = parsedMounts.group(1);
java.nio.file.Path srcPath = java.nio.file.Paths.get(src);
if (!srcPath.isAbsolute()) {
src = mountReadOnlyPath(src, localizedResources);
}
String dst = parsedMounts.group(2);
- String mode = parsedMounts.group(3);
- if (!mode.equals("ro") && !mode.equals("rw")) {
- throw new ContainerExecutionException(
- "Invalid mount mode requested for mount: "
- + parsedMounts.group());
- }
- if (mode.equals("ro")) {
- runCommand.addReadOnlyMountLocation(src, dst);
- } else {
- runCommand.addReadWriteMountLocation(src, dst);
+ String mode = parsedMounts.group(4);
+ if (mode == null) {
+ mode = "rw";
+ } else if (!mode.startsWith("ro") && !mode.startsWith("rw")) {
+ mode = "rw+" + mode;
}
+ runCommand.addMountLocation(src, dst, mode);
+ }
+ long commaCount = environment.get(ENV_DOCKER_CONTAINER_MOUNTS).chars()
+ .filter(c -> c == ',').count();
+ if (mountCount != commaCount + 1) {
+ // this means the matcher skipped an improperly formatted mount
+ throw new ContainerExecutionException(
+ "Unable to parse some mounts in user supplied mount list: "
+ + environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index af16178..48ada5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -65,19 +65,15 @@ public class DockerRunCommand extends DockerCommand {
}
public DockerRunCommand addMountLocation(String sourcePath, String
- destinationPath, boolean createSource) {
- boolean sourceExists = new File(sourcePath).exists();
- if (!sourceExists && !createSource) {
- return this;
- }
- super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ destinationPath, String mode) {
+ super.addCommandArguments("mounts", sourcePath + ":" +
+ destinationPath + ":" + mode);
return this;
}
public DockerRunCommand addReadWriteMountLocation(String sourcePath, String
destinationPath) {
- super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addMountLocation(sourcePath, destinationPath, "rw");
}
public DockerRunCommand addAllReadWriteMountLocations(List<String> paths) {
@@ -93,14 +89,12 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addReadOnlyMountLocation(sourcePath, destinationPath);
}
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
destinationPath) {
- super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
- return this;
+ return addMountLocation(sourcePath, destinationPath, "ro");
}
public DockerRunCommand addAllReadOnlyMountLocations(List<String> paths) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 580cd37..899c46a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -1095,105 +1095,150 @@ static char* get_mount_source(const char *mount) {
return strndup(mount, len);
}
-static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
- const int ro, args *args) {
- const char *ro_suffix = "";
+static char* get_mount_type(const char *mount) {
+ const char *tmp = strrchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ if (strlen(tmp) < 2) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ char *mount_type = strdup(&tmp[1]);
+ if (strncmp("ro", mount_type, 2) != 0 &&
+ strncmp("rw", mount_type, 2) != 0) {
+ fprintf(ERRORFILE, "Invalid docker mount type '%s'\n", mount_type);
+ free(mount_type);
+ return NULL;
+ }
+ if (strlen(mount_type) > 2) {
+ if (strlen(mount_type) < 8 ||
+ (strcmp("shared", mount_type + 3) != 0 &&
+ strcmp("rshared", mount_type + 3) != 0 &&
+ strcmp("slave", mount_type + 3) != 0 &&
+ strcmp("rslave", mount_type + 3) != 0 &&
+ strcmp("private", mount_type + 3) != 0 &&
+ strcmp("rprivate", mount_type + 3) != 0)) {
+ fprintf(ERRORFILE, "Invalid docker mount type '%s'\n", mount_type);
+ free(mount_type);
+ return NULL;
+ }
+ mount_type[2] = ',';
+ }
+ return mount_type;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
const char *tmp_path_buffer[2] = {NULL, NULL};
char *mount_src = NULL;
+ char *mount_type = NULL;
char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
- char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char **values = get_configuration_values_delimiter("mounts", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
char *config_path = get_config_path("");
const char *container_executor_cfg_path = normalize_mount(config_path, 0);
free(config_path);
int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
- if (ro != 0) {
- ro_suffix = ":ro";
+ if (values == NULL) {
+ goto free_and_exit;
}
- if (values != NULL) {
- // Disable mount volumes if image is not trusted.
- if (check_trusted_image(command_config, conf) != 0) {
- fprintf(ERRORFILE, "Disable mount volume for untrusted image\n");
- // YARN will implicitly bind node manager local directory to
- // docker image. This can create file system security holes,
- // if docker container has binary to escalate privileges.
- // For untrusted image, we drop mounting without reporting
- // INVALID_DOCKER_MOUNT messages to allow running untrusted
- // image in a sandbox.
- ret = 0;
+ // Disable mount volumes if image is not trusted.
+ if (check_trusted_image(command_config, conf) != 0) {
+ fprintf(ERRORFILE, "Disable mount volume for untrusted image\n");
+ // YARN will implicitly bind node manager local directory to
+ // docker image. This can create file system security holes,
+ // if docker container has binary to escalate privileges.
+ // For untrusted image, we drop mounting without reporting
+ // INVALID_DOCKER_MOUNT messages to allow running untrusted
+ // image in a sandbox.
+ ret = 0;
+ goto free_and_exit;
+ }
+ ret = normalize_mounts(permitted_ro_mounts, 1);
+ ret |= normalize_mounts(permitted_rw_mounts, 1);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; i++) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
+ ret = INVALID_DOCKER_MOUNT;
goto free_and_exit;
}
- ret = normalize_mounts(permitted_ro_mounts, 1);
- ret |= normalize_mounts(permitted_rw_mounts, 1);
- if (ret != 0) {
- fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
- ret = MOUNT_ACCESS_ERROR;
+ mount_type = get_mount_type(values[i]);
+ if (mount_type == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
+ ret = INVALID_DOCKER_MOUNT;
goto free_and_exit;
}
- for (i = 0; values[i] != NULL; i++) {
- mount_src = get_mount_source(values[i]);
- if (mount_src == NULL) {
- fprintf(ERRORFILE, "Invalid docker mount '%s'\n", values[i]);
- ret = INVALID_DOCKER_MOUNT;
- goto free_and_exit;
- }
- permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
- permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
- if (permitted_ro == -1 || permitted_rw == -1) {
- fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
- ret = INVALID_DOCKER_MOUNT;
- goto free_and_exit;
- }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ if (strncmp("rw", mount_type, 2) == 0) {
// rw mount
- if (ro == 0) {
- if (permitted_rw == 0) {
- fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src, 0);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
ret = INVALID_DOCKER_RW_MOUNT;
goto free_and_exit;
- } else {
- // determine if the user can modify the container-executor.cfg file
- tmp_path_buffer[0] = normalize_mount(mount_src, 0);
- // just re-use the function, flip the args to check if the container-executor path is in the requested
- // mount point
- ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
- free((void *) tmp_path_buffer[0]);
- if (ret == 1) {
- fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
- values[i]);
- ret = INVALID_DOCKER_RW_MOUNT;
- goto free_and_exit;
- }
}
}
- //ro mount
- if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ } else {
+ // ro mount
+ if (permitted_ro == 0 && permitted_rw == 0) {
fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
ret = INVALID_DOCKER_RO_MOUNT;
goto free_and_exit;
}
+ }
- ret = add_to_args(args, "-v");
- if (ret != 0) {
- ret = BUFFER_TOO_SMALL;
- goto free_and_exit;
- }
+ if (strlen(mount_type) > 2) {
+ // overwrite separator between read mode and propagation option with ','
+ int mount_type_index = strlen(values[i]) - strlen(mount_type);
+ values[i][mount_type_index + 2] = ',';
+ }
- char *tmp_buffer = make_string("%s%s", values[i], (char *) ro_suffix);
- ret = add_to_args(args, tmp_buffer);
- free(tmp_buffer);
- if (ret != 0) {
- ret = BUFFER_TOO_SMALL;
- goto free_and_exit;
- }
- free(mount_src);
- mount_src = NULL;
+ ret = add_to_args(args, "-v");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+
+ ret = add_to_args(args, values[i]);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
}
+ free(mount_src);
+ free(mount_type);
+ mount_src = NULL;
+ mount_type = NULL;
}
free_and_exit:
free(mount_src);
+ free(mount_type);
free_values(permitted_ro_mounts);
free_values(permitted_rw_mounts);
free_values(values);
@@ -1201,14 +1246,6 @@ free_and_exit:
return ret;
}
-static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
- return add_mounts(command_config, conf, "ro-mounts", 1, args);
-}
-
-static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, args *args) {
- return add_mounts(command_config, conf, "rw-mounts", 0, args);
-}
-
static int check_privileges(const char *user) {
int ngroups = 0;
gid_t *groups = NULL;
@@ -1427,12 +1464,7 @@ int get_docker_run_command(const char *command_file, const struct configuration
goto free_and_exit;
}
- ret = add_ro_mounts(&command_config, conf, args);
- if (ret != 0) {
- goto free_and_exit;
- }
-
- ret = add_rw_mounts(&command_config, conf, args);
+ ret = add_mounts(&command_config, conf, args);
if (ret != 0) {
goto free_and_exit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
index 007e737..7e18146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -934,36 +934,42 @@ namespace ContainerExecutor {
}
- TEST_F(TestDockerUtil, test_add_rw_mounts) {
+ TEST_F(TestDockerUtil, test_add_mounts) {
struct configuration container_cfg, cmd_cfg;
struct args buff = ARGS_INITIAL_VALUE;
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n "
- "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut\n "
+ "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,/usr/bin/awk\n "
"docker.allowed.ro-mounts=/etc/passwd";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var:/var", "-v /var:/var"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var:/var:rw", "-v /var:/var:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/var:/var", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/var:/var:rw", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var/:/var/", "-v /var/:/var/"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/:/var/:rw", "-v /var/:/var/:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/cut:/usr/bin/cut",
- "-v /usr/bin/cut:/usr/bin/cut"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/cut:/usr/bin/cut:rw",
+ "-v /usr/bin/cut:/usr/bin/cut:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/lib:/lib",
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/lib:/lib:rw",
""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
- "-v /opt:/mydisk1 -v /var/log/:/mydisk2"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/opt:/mydisk1:rw,/var/log/:/mydisk2:rw",
+ "-v /opt:/mydisk1:rw -v /var/log/:/mydisk2:rw"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2",
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/opt:/mydisk1:rw,/var/log/:/mydisk2:rw",
""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/awk:/awk:rw+shared,/etc/passwd:/etc/passwd:ro",
+ "-v /usr/bin/awk:/awk:rw,shared -v /etc/passwd:/etc/passwd:ro"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var:/var:ro+rprivate,/etc/passwd:/etc/passwd:ro+rshared",
+ "-v /var:/var:ro,rprivate -v /etc/passwd:/etc/passwd:ro,rshared"));
write_container_executor_cfg(container_executor_cfg_contents);
ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
if (ret != 0) {
@@ -984,7 +990,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
@@ -995,13 +1001,22 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/lib:/lib",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib:rw",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/:/usr/bin",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/:/usr/bin:rw",
static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/blah:/blah",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/blah:/blah:rw",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/tmp:/tmp:shared",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/lib:/lib:other",
static_cast<int>(INVALID_DOCKER_MOUNT)));
std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -1012,7 +1027,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
@@ -1024,14 +1039,14 @@ namespace ContainerExecutor {
// verify that you can't mount any directory in the container-executor.cfg path
char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
while (strlen(ce_path) != 0) {
- std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=";
- cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
+ std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=";
+ cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop").append(":rw");
write_command_file(cmd_file_contents);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
char *actual = flatten(&buff);
ASSERT_STREQ("", actual);
@@ -1046,7 +1061,7 @@ namespace ContainerExecutor {
free(ce_path);
free_configuration(&container_cfg);
- // For untrusted image, container add_rw_mounts will pass through
+ // For untrusted image, container add_mounts will pass through
// without mounting or report error code.
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@@ -1054,7 +1069,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ("", actual);
@@ -1073,26 +1088,26 @@ namespace ContainerExecutor {
"docker.allowed.ro-mounts=/etc/passwd,/etc/group";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/var:/var", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/var:/var:ro", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n ro-mounts=/etc:/etc", ""));
+ "[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n mounts=/etc:/etc:ro", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/:/var/", "-v /var/:/var/:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/:/var/:ro", "-v /var/:/var/:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home", "-v /home:/home:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home:/home:ro", "-v /home:/home:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home/:/home", "-v /home/:/home:ro"));
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home/:/home:ro", "-v /home/:/home:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/usr/bin/cut:/usr/bin/cut",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/usr/bin/cut:/usr/bin/cut:ro",
"-v /usr/bin/cut:/usr/bin/cut:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/group:/etc/group",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc/group:/etc/group:ro",
"-v /etc/group:/etc/group:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/passwd:/etc/passwd",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc/passwd:/etc/passwd:ro",
"-v /etc/passwd:/etc/passwd:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/var/log:/mydisk1:ro,/etc/passwd:/etc/passwd:ro",
"-v /var/log:/mydisk1:ro -v /etc/passwd:/etc/passwd:ro"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", ""));
@@ -1116,7 +1131,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
@@ -1127,10 +1142,10 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc:/etc",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/etc:/etc:ro",
static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
- "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/blah:/blah",
+ "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/blah:/blah:ro",
static_cast<int>(INVALID_DOCKER_MOUNT)));
std::vector<std::pair<std::string, int> >::const_iterator itr2;
@@ -1141,7 +1156,7 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
@@ -1157,12 +1172,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
- write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home");
+ write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n mounts=/home:/home:ro");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
- ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
+ ret = add_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
@@ -1203,18 +1218,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
" --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash "
"test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1225,18 +1240,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1247,24 +1262,24 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image "
"bash test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --privileged --cap-drop=ALL "
"--cap-add=CHOWN --cap-add=SETUID --hostname=host-id "
"--device=/dev/test:/dev/test hadoop/docker-image bash test_script.sh arg1 arg2"));
@@ -1292,7 +1307,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1302,7 +1317,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n"
+ " mounts=/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/var/log:/var/log:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1312,7 +1327,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/bin:/bin:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1322,7 +1337,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1332,7 +1347,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1342,7 +1357,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1432,18 +1447,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN"
" --cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash "
"test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1453,18 +1468,18 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
"run --name=container_e1_12312_11111_02_000001 --user=nobody -d --rm --net=bridge -v /var/log:/var/log:ro -v /var/lib:/lib:ro"
- " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
+ " -v /usr/bin/cut:/usr/bin/cut:ro -v /tmp:/tmp:rw --cgroup-parent=ctr-cgroup --cap-drop=ALL --cap-add=CHOWN "
"--cap-add=SETUID --hostname=host-id --device=/dev/test:/dev/test hadoop/docker-image bash"
" test_script.sh arg1 arg2"));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=nothadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n net=bridge\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
@@ -1475,7 +1490,7 @@ namespace ContainerExecutor {
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
"[docker-command-execution]\n"
" docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=nobody\n hostname=host-id\n"
- " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n"
+ " mounts=/var/log:/var/log:ro,/var/lib:/lib:ro,/usr/bin/cut:/usr/bin/cut:ro,/tmp:/tmp:rw\n"
" network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
" cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
" launch-command=bash,test_script.sh,arg1,arg2",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 855ec44..80823d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -394,7 +394,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -411,17 +411,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -445,7 +444,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals(13, dockerCommands.size());
int counter = 0;
Assert.assertEquals("[docker-command-execution]",
dockerCommands.get(counter++));
@@ -461,18 +460,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -560,7 +558,7 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -579,18 +577,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=" + allowedNetwork, dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -620,7 +617,7 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -639,18 +636,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert
.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -690,7 +686,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -711,16 +707,16 @@ public class TestDockerContainerRuntime {
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
Assert.assertEquals(
- " name=container_e11_1518975676334_14532816_01_000001",
- dockerCommands.get(counter++));
- Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
+ " mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
dockerCommands.get(counter++));
Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
+ " name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -759,18 +755,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
-
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=sdn2", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -808,7 +802,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -859,7 +853,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -876,18 +870,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" pid=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -912,7 +905,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -1021,7 +1014,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1036,18 +1029,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" privileged=true", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + submittingUser,
dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
@@ -1108,7 +1100,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source");
+ "/source");
try {
runtime.launchContainer(builder.build());
@@ -1138,7 +1130,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1155,19 +1147,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/test_local_dir/test_resource_file:test_mount:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(
- " ro-mounts=/test_filecache_dir:/test_filecache_dir,/"
- + "test_user_filecache_dir:/test_user_filecache_dir,"
- + "/test_local_dir/test_resource_file:test_mount",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1194,7 +1184,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1211,20 +1201,18 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/test_local_dir/test_resource_file:test_mount1:ro,"
+ + "/test_local_dir/test_resource_file:test_mount2:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(
- " ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/test_local_dir/test_resource_file:test_mount1,"
- + "/test_local_dir/test_resource_file:test_mount2",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1240,7 +1228,8 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw");
+ "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw,/tmp/baz:/tmp/baz," +
+ "/a:/a:shared,/b:/b:ro+shared,/c:/c:rw+rshared,/d:/d:private");
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@@ -1250,7 +1239,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1267,19 +1256,19 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:ro,"
+ + "/tmp/bar:/tmp/bar:rw,/tmp/baz:/tmp/baz:rw,/a:/a:rw+shared,"
+ + "/b:/b:ro+shared,/c:/c:rw+rshared,/d:/d:rw+private",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/tmp/foo:/tmp/foo",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir,"
- + "/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1293,7 +1282,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source:target");
+ "/source:target:ro,/source:target:other,/source:target:rw");
try {
runtime.launchContainer(builder.build());
@@ -1311,7 +1300,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "source:target:other");
+ "/source:target:other");
try {
runtime.launchContainer(builder.build());
@@ -1329,7 +1318,7 @@ public class TestDockerContainerRuntime {
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
- "s\0ource:target:ro");
+ "/s\0ource:target:ro");
try {
runtime.launchContainer(builder.build());
@@ -1357,7 +1346,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1374,18 +1363,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -1425,7 +1413,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 14;
+ int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -1442,18 +1430,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/tmp/foo:/tmp/foo:rw,/tmp/bar:/tmp/bar:rw",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir,"
- + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
@@ -2012,7 +1999,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -2029,18 +2016,17 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro,"
+ + "/source/path:/destination/path:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir,"
- + "/source/path:/destination/path",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
// Verify volume-driver is set to expected value.
@@ -2153,7 +2139,7 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- int expected = 15;
+ int expected = 14;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
@@ -2172,17 +2158,16 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
+ Assert.assertEquals(" mounts="
+ + "/test_container_log_dir:/test_container_log_dir:rw,"
+ + "/test_application_local_dir:/test_application_local_dir:rw,"
+ + "/test_filecache_dir:/test_filecache_dir:ro,"
+ + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+ dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
- Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir,"
- + "/test_user_filecache_dir:/test_user_filecache_dir",
- dockerCommands.get(counter++));
- Assert.assertEquals(
- " rw-mounts=/test_container_log_dir:/test_container_log_dir,"
- + "/test_application_local_dir:/test_application_local_dir",
- dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter++));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
index 7057847..c8b2eaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/TestNvidiaDockerV1CommandPlugin.java
@@ -196,7 +196,7 @@ public class TestNvidiaDockerV1CommandPlugin {
// Volume driver should not be included by final commandline
Assert.assertFalse(newCommandLine.containsKey("volume-driver"));
Assert.assertTrue(newCommandLine.containsKey("devices"));
- Assert.assertTrue(newCommandLine.containsKey("ro-mounts"));
+ Assert.assertTrue(newCommandLine.containsKey("mounts"));
/* Test get docker volume command */
commandPlugin = new MyNvidiaDockerV1CommandPlugin(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8688a0c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index a2ef6fe..e35c906 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -304,7 +304,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE` | Controls which PID namespace will be used by the Docker container. By default, each Docker container has its own PID namespace. To share the namespace of the host, the yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed property must be set to true. If the host PID namespace is allowed and this environment variable is set to host, the Docker container will share the host's PID namespace. No other value is allowed. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
-| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
+| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as `source:dest[:mode]` and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. If neither is specified, read-write will be assumed. The mode may include a bind propagation option. In that case, the mode should either be of the form `[option]`, `rw+[option]`, or `ro+[option]`. Valid bind propagation options are shared, rshared, slave, rslave, private, and rprivate. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts`. |
| `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. |
The first two are required. The remainder can be set as needed. While
@@ -347,10 +347,13 @@ supplied by the user must either match or be a child of the specified
directory.
The user supplied mount list is defined as a comma separated list in the form
-*source*:*destination*:*mode*. The source is the file or directory on the host.
-The destination is the path within the contatiner where the source will be bind
-mounted. The mode defines the mode the user expects for the mount, which can be
-ro (read-only) or rw (read-write).
+*source*:*destination* or *source*:*destination*:*mode*. The source is the file
+or directory on the host. The destination is the path within the container
+where the source will be bind mounted. The mode defines the mode the user
+expects for the mount, which can be ro (read-only) or rw (read-write). If not
+specified, rw is assumed. The mode may also include a bind propagation option
+ (shared, rshared, slave, rslave, private, or rprivate). In that case, the
+ mode should be of the form *option*, rw+*option*, or ro+*option*.
The following example outlines how to use this feature to mount the commonly
needed /sys/fs/cgroup directory into the container running on YARN.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org