You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/09/28 23:46:01 UTC
[3/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
new file mode 100644
index 0000000..860320d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -0,0 +1,998 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <ctype.h>
+#include "../modules/common/module-configs.h"
+#include "docker-util.h"
+#include "string-utils.h"
+#include "util.h"
+
+static int read_and_verify_command_file(const char *command_file, const char *docker_command,
+ struct configuration *command_config) {
+ int ret = 0;
+ ret = read_config(command_file, command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (command == NULL || (strcmp(command, docker_command) != 0)) {
+ ret = INCORRECT_COMMAND;
+ }
+ free(command);
+ return ret;
+}
+
+static int add_to_buffer(char *buff, const size_t bufflen, const char *string) {
+ size_t current_len = strlen(buff);
+ size_t string_len = strlen(string);
+ if (current_len + string_len < bufflen - 1) {
+ strncpy(buff + current_len, string, string_len);
+ buff[current_len + string_len] = '\0';
+ return 0;
+ }
+ return -1;
+}
+
+static int add_param_to_command(const struct configuration *command_config, const char *key, const char *param,
+ const int with_argument, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ int ret = 0;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (value != NULL) {
+ if (with_argument) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, value);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ } else if (strcmp(value, "true") == 0) {
+ ret = add_to_buffer(out, outlen, param);
+ }
+ free(value);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ }
+ free(tmp_buffer);
+ return ret;
+}
+
+static int add_param_to_command_if_allowed(const struct configuration *command_config,
+ const struct configuration *executor_cfg,
+ const char *key, const char *allowed_key, const char *param,
+ const int multiple_values, const char prefix,
+ char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *tmp_ptr = NULL;
+ char **values = NULL;
+ char **permitted_values = get_configuration_values_delimiter(allowed_key,
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, executor_cfg,
+ ",");
+ int i = 0, j = 0, permitted = 0, ret = 0;
+ if (multiple_values) {
+ values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ } else {
+ values = (char **) alloc_and_clear_memory(2, sizeof(char *));
+ values[0] = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ values[1] = NULL;
+ if (values[0] == NULL) {
+ ret = 0;
+ goto free_and_exit;
+ }
+ }
+
+ if (values != NULL) {
+ if (permitted_values != NULL) {
+ for (i = 0; values[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ permitted = 0;
+ if(prefix != 0) {
+ tmp_ptr = strchr(values[i], prefix);
+ if (tmp_ptr == NULL) {
+ fprintf(ERRORFILE, "Prefix char '%c' not found in '%s'\n",
+ prefix, values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ for (j = 0; permitted_values[j] != NULL; ++j) {
+ if (prefix == 0) {
+ ret = strcmp(values[i], permitted_values[j]);
+ } else {
+ ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
+ }
+ if (ret == 0) {
+ permitted = 1;
+ break;
+ }
+ }
+ if (permitted == 1) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, values[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Output buffer too small\n");
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested\n", values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested, "
+ "permitted values list is empty\n", values[0]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free_values(values);
+ free_values(permitted_values);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_docker_config_param(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "docker-config", "--config=", 1, out, outlen);
+}
+
+static int validate_container_name(const char *container_name) {
+ const char *CONTAINER_NAME_PREFIX = "container_";
+ if (0 == strncmp(container_name, CONTAINER_NAME_PREFIX, strlen(CONTAINER_NAME_PREFIX))) {
+ if (1 == validate_container_id(container_name)) {
+ return 0;
+ }
+ }
+ fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", container_name);
+ fflush(ERRORFILE);
+ return INVALID_DOCKER_CONTAINER_NAME;
+}
+
+const char *get_docker_error_message(const int error_code) {
+
+ switch (error_code) {
+ case INVALID_COMMAND_FILE:
+ return "Invalid command file passed";
+ case INCORRECT_COMMAND:
+ return "Incorrect command";
+ case BUFFER_TOO_SMALL:
+ return "Command buffer too small";
+ case INVALID_DOCKER_CONTAINER_NAME:
+ return "Invalid docker container name";
+ case INVALID_DOCKER_IMAGE_NAME:
+ return "Invalid docker image name";
+ case INVALID_DOCKER_USER_NAME:
+ return "Invalid docker user name";
+ case INVALID_DOCKER_INSPECT_FORMAT:
+ return "Invalid docker inspect format";
+ case UNKNOWN_DOCKER_COMMAND:
+ return "Unknown docker command";
+ case INVALID_DOCKER_NETWORK:
+ return "Invalid docker network";
+ case INVALID_DOCKER_CAPABILITY:
+ return "Invalid docker capability";
+ case PRIVILEGED_CONTAINERS_DISABLED:
+ return "Privileged containers are disabled";
+ case INVALID_DOCKER_MOUNT:
+ return "Invalid docker mount";
+ case INVALID_DOCKER_RO_MOUNT:
+ return "Invalid docker read-only mount";
+ case INVALID_DOCKER_RW_MOUNT:
+ return "Invalid docker read-write mount";
+ case MOUNT_ACCESS_ERROR:
+ return "Mount access error";
+ case INVALID_DOCKER_DEVICE:
+ return "Invalid docker device";
+ default:
+ return "Unknown error";
+ }
+}
+
+char *get_docker_binary(const struct configuration *conf) {
+ char *docker_binary = NULL;
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (docker_binary == NULL) {
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, "", conf);
+ if (docker_binary == NULL) {
+ docker_binary = strdup("/usr/bin/docker");
+ }
+ }
+ return docker_binary;
+}
+
+int docker_module_enabled(const struct configuration *conf) {
+ struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (section != NULL) {
+ return module_enabled(section, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION);
+ }
+ return 0;
+}
+
+int get_docker_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ struct configuration command_config = {0, NULL};
+ ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
+ return get_docker_inspect_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
+ return get_docker_load_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
+ return get_docker_pull_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
+ return get_docker_rm_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
+ return get_docker_run_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
+ return get_docker_stop_command(command_file, conf, out, outlen);
+ } else {
+ return UNKNOWN_DOCKER_COMMAND;
+ }
+}
+
+int get_docker_inspect_command(const char *command_file, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ const char *valid_format_strings[] = { "{{.State.Status}}",
+ "{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}" };
+ int ret = 0, i = 0, valid_format = 0;
+ char *format = NULL, *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (format == NULL) {
+ free(container_name);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+ for (i = 0; i < 2; ++i) {
+ if (strcmp(format, valid_format_strings[i]) == 0) {
+ valid_format = 1;
+ break;
+ }
+ }
+ if (valid_format != 1) {
+ fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
+ free(container_name);
+ free(format);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(container_name);
+ free(format);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_INSPECT_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " --format=");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, format);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ free(format);
+ free(container_name);
+ return 0;
+
+ free_and_exit:
+ free(format);
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_load_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_LOAD_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --i=", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int validate_docker_image_name(const char *image_name) {
+ const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+ return execute_regex_match(regex_str, image_name);
+}
+
+int get_docker_pull_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_PULL_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " ", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_rm_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RM_COMMAND);
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_stop_command(const char *command_file, const struct configuration *conf,
+ char *out, const size_t outlen) {
+ int ret = 0;
+ size_t len = 0, i = 0;
+ char *value = NULL;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_STOP_COMMAND);
+ if (ret == 0) {
+ value = get_configuration_value("time", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL) {
+ len = strlen(value);
+ for (i = 0; i < len; ++i) {
+ if (isdigit(value[i]) == 0) {
+ fprintf(ERRORFILE, "Value for time is not a number '%s'\n", value);
+ free(container_name);
+ memset(out, 0, outlen);
+ return INVALID_DOCKER_STOP_COMMAND;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " --time=");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, value);
+ }
+ if (ret != 0) {
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int detach_container(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "detach", "-d ", 0, out, outlen);
+}
+
+static int rm_container_on_exit(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "rm", "--rm ", 0, out, outlen);
+}
+
+static int set_container_workdir(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "workdir", "--workdir=", 1, out, outlen);
+}
+
+static int set_cgroup_parent(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "cgroup-parent", "--cgroup-parent=", 1, out, outlen);
+}
+
+static int set_hostname(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "hostname", "--hostname=", 1, out, outlen);
+}
+
+static int set_group_add(const struct configuration *command_config, char *out, const size_t outlen) {
+ int i = 0, ret = 0;
+ char **group_add = get_configuration_values_delimiter("group-add", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = NULL;
+
+ if (group_add != NULL) {
+ for (i = 0; group_add[i] != NULL; ++i) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--group-add ", group_add[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ }
+ return ret;
+}
+
+static int set_network(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "net",
+ "docker.allowed.networks", "--net=",
+ 0, 0, out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Could not find requested network in allowed networks\n");
+ ret = INVALID_DOCKER_NETWORK;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_capabilities(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+
+ ret = add_to_buffer(out, outlen, "--cap-drop='ALL' ");
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ ret = add_param_to_command_if_allowed(command_config, conf, "cap-add",
+ "docker.allowed.capabilities",
+ "--cap-add=", 1, 0,
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker capability requested\n");
+ ret = INVALID_DOCKER_CAPABILITY;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_devices(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "devices", "docker.allowed.devices", "--device=", 1, ':',
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker device requested\n");
+ ret = INVALID_DOCKER_DEVICE;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+/**
+ * Helper function to help normalize mounts for checking if mounts are
+ * permitted. The function does the following -
+ * 1. Find the canonical path for mount using realpath
+ * 2. If the path is a directory, add a '/' at the end (if not present)
+ * 3. Return a copy of the canonicalised path(to be freed by the caller)
+ * @param mount path to be canonicalised
+ * @return pointer to canonicalised path, NULL on error
+ */
+static char* normalize_mount(const char* mount) {
+ int ret = 0;
+ struct stat buff;
+ char *ret_ptr = NULL, *real_mount = NULL;
+ if (mount == NULL) {
+ return NULL;
+ }
+ real_mount = realpath(mount, NULL);
+ if (real_mount == NULL) {
+ fprintf(ERRORFILE, "Could not determine real path of mount '%s'\n", mount);
+ free(real_mount);
+ return NULL;
+ }
+ ret = stat(real_mount, &buff);
+ if (ret == 0) {
+ if (S_ISDIR(buff.st_mode)) {
+ size_t len = strlen(real_mount);
+ if (len <= 0) {
+ return NULL;
+ }
+ if (real_mount[len - 1] != '/') {
+ ret_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char));
+ strncpy(ret_ptr, real_mount, len);
+ ret_ptr[len] = '/';
+ ret_ptr[len + 1] = '\0';
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ fprintf(ERRORFILE, "Could not stat path '%s'\n", real_mount);
+ ret_ptr = NULL;
+ }
+ free(real_mount);
+ return ret_ptr;
+}
+
+static int normalize_mounts(char **mounts) {
+ int i = 0;
+ char *tmp = NULL;
+ if (mounts == NULL) {
+ return 0;
+ }
+ for (i = 0; mounts[i] != NULL; ++i) {
+ tmp = normalize_mount(mounts[i]);
+ if (tmp == NULL) {
+ return -1;
+ }
+ free(mounts[i]);
+ mounts[i] = tmp;
+ }
+ return 0;
+}
+
+static int check_mount_permitted(const char **permitted_mounts, const char *requested) {
+ int i = 0, ret = 0;
+ size_t permitted_mount_len = 0;
+ char *normalized_path = normalize_mount(requested);
+ if (permitted_mounts == NULL) {
+ return 0;
+ }
+ if (normalized_path == NULL) {
+ return -1;
+ }
+ for (i = 0; permitted_mounts[i] != NULL; ++i) {
+ if (strcmp(normalized_path, permitted_mounts[i]) == 0) {
+ ret = 1;
+ break;
+ }
+ // directory check
+ permitted_mount_len = strlen(permitted_mounts[i]);
+ if (permitted_mount_len > 0
+ && permitted_mounts[i][permitted_mount_len - 1] == '/') {
+ if (strncmp(normalized_path, permitted_mounts[i], permitted_mount_len) == 0) {
+ ret = 1;
+ break;
+ }
+ }
+
+ }
+ free(normalized_path);
+ return ret;
+}
+
+static char* get_mount_source(const char *mount) {
+ char *src_mount = NULL;
+ const char *tmp = NULL;
+ tmp = strchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ src_mount = strndup(mount, tmp - mount);
+ return src_mount;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
+ const int ro, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ const char *ro_suffix = "";
+ const char *tmp_path_buffer[2] = {NULL, NULL};
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char *tmp_buffer_2 = NULL, *mount_src = NULL;
+ const char *container_executor_cfg_path = normalize_mount(get_config_path(""));
+ int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
+ if (ro != 0) {
+ ro_suffix = ":ro";
+ }
+
+ if (values != NULL) {
+ ret = normalize_mounts(permitted_ro_mounts);
+ ret |= normalize_mounts(permitted_rw_mounts);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; ++i) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ // rw mount
+ if (ro == 0) {
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ }
+ }
+ }
+ //ro mount
+ if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RO_MOUNT;
+ goto free_and_exit;
+ }
+ tmp_buffer_2 = (char *) alloc_and_clear_memory(strlen(values[i]) + strlen(ro_suffix) + 1, sizeof(char));
+ strncpy(tmp_buffer_2, values[i], strlen(values[i]));
+ strncpy(tmp_buffer_2 + strlen(values[i]), ro_suffix, strlen(ro_suffix));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "-v ", tmp_buffer_2);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer_2);
+ free(mount_src);
+ tmp_buffer_2 = NULL;
+ mount_src = NULL;
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ }
+
+ free_and_exit:
+ free_values(permitted_ro_mounts);
+ free_values(permitted_rw_mounts);
+ free_values(values);
+ free(mount_src);
+ free((void *) container_executor_cfg_path);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "ro-mounts", 1, out, outlen);
+}
+
+static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
+}
+
+static int set_privileged(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config);
+ char *privileged_container_enabled
+ = get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ int ret = 0;
+
+ if (value != NULL && strcmp(value, "true") == 0) {
+ if (privileged_container_enabled != NULL) {
+ if (strcmp(privileged_container_enabled, "1") == 0) {
+ ret = add_to_buffer(out, outlen, "--privileged ");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free(tmp_buffer);
+ free(value);
+ free(privileged_container_enabled);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+int get_docker_run_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0, i = 0;
+ char *container_name = NULL, *user = NULL, *image = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+ user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (user == NULL) {
+ return INVALID_DOCKER_USER_NAME;
+ }
+ image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image == NULL || validate_docker_image_name(image) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RUN_COMMAND);
+ if(ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --name=", container_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--user=", user);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ ret = detach_container(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = rm_container_on_exit(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_container_workdir(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_network(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_ro_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_rw_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_cgroup_parent(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_privileged(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_capabilities(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_hostname(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_group_add(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_devices(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", image);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", launch_command[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ free_values(launch_command);
+ free(tmp_buffer);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ free_values(launch_command);
+ }
+ free(tmp_buffer);
+ return 0;
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
new file mode 100644
index 0000000..37ec880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+
+#include "configuration.h"
+
+#define CONTAINER_EXECUTOR_CFG_DOCKER_SECTION "docker"
+#define DOCKER_BINARY_KEY "docker.binary"
+#define DOCKER_COMMAND_FILE_SECTION "docker-command-execution"
+#define DOCKER_INSPECT_COMMAND "inspect"
+#define DOCKER_LOAD_COMMAND "load"
+#define DOCKER_PULL_COMMAND "pull"
+#define DOCKER_RM_COMMAND "rm"
+#define DOCKER_RUN_COMMAND "run"
+#define DOCKER_STOP_COMMAND "stop"
+
+
+enum docker_error_codes {
+ INVALID_COMMAND_FILE = 1,
+ INCORRECT_COMMAND,
+ BUFFER_TOO_SMALL,
+ INVALID_DOCKER_CONTAINER_NAME,
+ INVALID_DOCKER_IMAGE_NAME,
+ INVALID_DOCKER_USER_NAME,
+ INVALID_DOCKER_INSPECT_FORMAT,
+ UNKNOWN_DOCKER_COMMAND,
+ INVALID_DOCKER_NETWORK,
+ INVALID_DOCKER_CAPABILITY,
+ PRIVILEGED_CONTAINERS_DISABLED,
+ INVALID_DOCKER_MOUNT,
+ INVALID_DOCKER_RO_MOUNT,
+ INVALID_DOCKER_RW_MOUNT,
+ MOUNT_ACCESS_ERROR,
+ INVALID_DOCKER_DEVICE,
+ INVALID_DOCKER_STOP_COMMAND
+};
+
+/**
+ * Get the full path for the docker binary.
+ * @param conf Configuration for the container-executor
+ * @return String containing the docker binary to be freed by the user.
+ */
+char *get_docker_binary(const struct configuration *conf);
+
+/**
+ * Get the Docker command line string. The function will inspect the params file to determine the command to be run.
+ * @param command_file File containing the params for the Docker command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the Docker command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker inspect command line string. The function will verify that the params file is meant for the
+ * inspect command.
+ * @param command_file File containing the params for the Docker inspect command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the inspect command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_inspect_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker load command line string. The function will verify that the params file is meant for the load command.
+ * @param command_file File containing the params for the Docker load command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the load command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_load_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker pull command line string. The function will verify that the params file is meant for the pull command.
+ * @param command_file File containing the params for the Docker pull command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the pull command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_pull_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker rm command line string. The function will verify that the params file is meant for the rm command.
+ * @param command_file File containing the params for the Docker rm command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the rm command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_rm_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker run command line string. The function will verify that the params file is meant for the run command.
+ * @param command_file File containing the params for the Docker run command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the run command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_run_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker stop command line string. The function will verify that the params file is meant for the stop command.
+ * @param command_file File containing the params for the Docker stop command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the stop command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_stop_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Give an error message for the supplied error code
+ * @param error_code the error code
+ * @return const string containing the error message
+ */
+const char *get_docker_error_message(const int error_code);
+
+/**
+ * Determine if the docker module is enabled in the config.
+ * @param conf Configuration structure
+ * @return 1 if enabled, 0 otherwise
+ */
+int docker_module_enabled(const struct configuration *conf);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index d19c084..40b2c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -21,7 +21,6 @@
#include <errno.h>
#include <strings.h>
#include <string.h>
-#include <stdio.h>
#include <stdlib.h>
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
new file mode 100644
index 0000000..339bec9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[docker]
+privieged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 64ee717..609dd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -19,6 +19,7 @@
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
+#include "get_executable.h"
#include "test/test-container-executor-common.h"
#include <inttypes.h>
@@ -1155,145 +1156,6 @@ void test_trim_function() {
free(trimmed);
}
-void test_sanitize_docker_command() {
-
- char *input[] = {
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run ''''''''",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
- char *expected_output[] = {
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
-
- int input_size = sizeof(input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < input_size; i++) {
- char *command = (char *) calloc(strlen(input[i]) + 1 , sizeof(char));
- strncpy(command, input[i], strlen(input[i]));
- char *op = sanitize_docker_command(command);
- if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
- printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
- exit(1);
- }
- free(command);
- }
-}
-
-void test_validate_docker_image_name() {
-
- char *good_input[] = {
- "ubuntu",
- "ubuntu:latest",
- "ubuntu:14.04",
- "ubuntu:LATEST",
- "registry.com:5000/user/ubuntu",
- "registry.com:5000/user/ubuntu:latest",
- "registry.com:5000/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu",
- "registry.com/user/ubuntu:latest",
- "registry.com/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu:test-image",
- "registry.com/user/ubuntu:test_image",
- "registry.com/ubuntu",
- "user/ubuntu",
- "user/ubuntu:0.1.2.3",
- "user/ubuntu:latest",
- "user/ubuntu:test_image",
- "user/ubuntu.test:test_image",
- "user/ubuntu-test:test-image",
- "registry.com/ubuntu/ubuntu/ubuntu"
- };
-
- char *bad_input[] = {
- "UBUNTU",
- "registry.com|5000/user/ubuntu",
- "registry.com | 5000/user/ubuntu",
- "ubuntu' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''",
- "bad_host_name:5000/user/ubuntu",
- "registry.com:foo/ubuntu/ubuntu/ubuntu",
- "registry.com/ubuntu:foo/ubuntu/ubuntu"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_docker_image_name(good_input[i]);
- if(0 != op) {
- printf("\nFAIL: docker image name %s is invalid", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_docker_image_name(bad_input[j]);
- if(1 != op) {
- printf("\nFAIL: docker image name %s is valid, expected invalid", bad_input[j]);
- exit(1);
- }
- }
-}
-
-void test_validate_container_id() {
- char *good_input[] = {
- "container_e134_1499953498516_50875_01_000007",
- "container_1499953498516_50875_01_000007",
- "container_e1_12312_11111_02_000001"
- };
-
- char *bad_input[] = {
- "CONTAINER",
- "container_e1_12312_11111_02_000001 | /tmp/file"
- "container_e1_12312_11111_02_000001 || # /tmp/file",
- "container_e1_12312_11111_02_000001 # /tmp/file",
- "container_e1_12312_11111_02_000001' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_container_id(good_input[i]);
- if(1 != op) {
- printf("FAIL: docker container name %s is invalid\n", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_container_id(bad_input[j]);
- if(0 != op) {
- printf("FAIL: docker container name %s is valid, expected invalid\n", bad_input[j]);
- exit(1);
- }
- }
-}
-
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1388,15 +1250,6 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
- printf("\nTesting sanitize docker commands()\n");
- test_sanitize_docker_command();
-
- printf("\nTesting validate_docker_image_name()\n");
- test_validate_docker_image_name();
-
- printf("\nTesting validate_container_id()\n");
- test_validate_container_id();
-
test_check_user(0);
#ifdef __APPLE__
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
index 2ec7b2a..b96dea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
@@ -17,7 +17,7 @@
*/
#include <gtest/gtest.h>
-#include <sstream>
+#include <vector>
extern "C" {
#include "util.h"
@@ -135,4 +135,39 @@ namespace ContainerExecutor {
ASSERT_STREQ("foo", trimmed);
free(trimmed);
}
+
+ TEST_F(TestUtil, test_escape_single_quote) {
+ std::vector<std::pair<std::string, std::string> > input_output_vec;
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'abcd'", "'\"'\"'abcd'\"'\"'"));
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'", "'\"'\"'"));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = input_output_vec.begin(); itr != input_output_vec.end(); ++itr) {
+ char *ret = escape_single_quote(itr->first.c_str());
+ ASSERT_STREQ(itr->second.c_str(), ret);
+ free(ret);
+ }
+ }
+
+ TEST_F(TestUtil, test_quote_and_append_arg) {
+
+ char *tmp = static_cast<char *>(malloc(4096));
+ size_t tmp_size = 4096;
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "ab'cd");
+ ASSERT_STREQ("param='ab'\"'\"'cd' ", tmp);
+ free(tmp);
+
+ tmp = static_cast<char *>(malloc(4));
+ tmp_size = 4;
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+ ASSERT_EQ(1040, tmp_size);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
index 037816a..b259c6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -88,6 +88,39 @@
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
-}
+ }
+
+ TEST_F(TestStringUtils, test_validate_container_id) {
+
+ const char *good_input[] = {
+ "container_e134_1499953498516_50875_01_000007",
+ "container_1499953498516_50875_01_000007",
+ "container_e1_12312_11111_02_000001"
+ };
+
+ const char *bad_input[] = {
+ "CONTAINER",
+ "container_e1_12312_11111_02_000001 | /tmp/file"
+ "container_e1_12312_11111_02_000001 || # /tmp/file",
+ "container_e1_12312_11111_02_000001 # /tmp/file",
+ "container_e1_12312_11111_02_000001' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_container_id(good_input[i]);
+ ASSERT_EQ(1, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_container_id(bad_input[j]);
+ ASSERT_EQ(0, op);
+ }
+ }
-} // namespace ContainerExecutor
\ No newline at end of file
+} // namespace ContainerExecutor
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org