You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/10/09 08:04:16 UTC
hadoop git commit: YARN-5704. Provide config knobs to control
enabling/disabling new/work in progress features in container-executor.
Contributed by Sidharta Seethana.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 50409404b -> d1841b475
YARN-5704. Provide config knobs to control enabling/disabling new/work in progress features in container-executor. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1841b47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1841b47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1841b47
Branch: refs/heads/branch-2.8
Commit: d1841b475002460239d25364c21dc6f6a3106092
Parents: 5040940
Author: Varun Vasudev <vv...@apache.org>
Authored: Sun Oct 9 13:22:01 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Sun Oct 9 13:22:01 2016 +0530
----------------------------------------------------------------------
.../impl/container-executor.c | 44 +++-
.../impl/container-executor.h | 11 +-
.../main/native/container-executor/impl/main.c | 258 +++++++++++++------
3 files changed, 221 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1841b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index a10c8d4..0d30bc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -51,6 +51,9 @@ static const char* TC_MODIFY_STATE_OPTS [] = { "-b" , NULL};
static const char* TC_READ_STATE_OPTS [] = { "-b", NULL};
static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
+static const int DEFAULT_DOCKER_SUPPORT_ENABLED = 0;
+static const int DEFAULT_TC_SUPPORT_ENABLED = 0;
+
//struct to store the user details
struct passwd *user_detail = NULL;
@@ -973,6 +976,41 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) {
}
+static int is_feature_enabled(const char* feature_key, int default_value) {
+ char *enabled_str = get_value(feature_key, &executor_cfg);
+ int enabled = default_value;
+
+ if (enabled_str != NULL) {
+ char *end_ptr = NULL;
+ enabled = strtol(enabled_str, &end_ptr, 10);
+
+ if ((enabled_str == end_ptr || *end_ptr != '\0') ||
+ (enabled < 0 || enabled > 1)) {
+ fprintf(LOGFILE, "Illegal value '%s' for '%s' in configuration. "
+ "Using default value: %d.\n", enabled_str, feature_key,
+ default_value);
+ fflush(LOGFILE);
+ free(enabled_str);
+ return default_value;
+ }
+
+ free(enabled_str);
+ return enabled;
+ } else {
+ return default_value;
+ }
+}
+
+
+int is_docker_support_enabled() {
+ return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
+ DEFAULT_DOCKER_SUPPORT_ENABLED);
+}
+
+int is_tc_support_enabled() {
+ return is_feature_enabled(TC_SUPPORT_ENABLED_KEY,
+ DEFAULT_TC_SUPPORT_ENABLED);
+}
/**
* Function to prepare the application directories for the container.
*/
@@ -1094,9 +1132,6 @@ int run_docker(const char *command_file) {
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
char **args = extract_values_delim(docker_command_with_binary, " ");
- //clean up command file before we exec
- unlink(command_file);
-
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
@@ -1407,8 +1442,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
cleanup:
- //clean up docker command file
- unlink(command_file);
if (exit_code_file != NULL && write_exit_code_file(exit_code_file, exit_code) < 0) {
fprintf (ERRORFILE,
@@ -1878,7 +1911,6 @@ static int run_traffic_control(const char *opts[], char *command_file) {
fprintf(LOGFILE, "failed to execute tc command!\n");
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
- unlink(command_file);
return 0;
} else {
execv(TC_BIN, (char**)args);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1841b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index fbab6bb..a80bfdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,7 +59,8 @@ enum errorcodes {
TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
DOCKER_RUN_FAILED=29,
ERROR_OPENING_FILE = 30,
- ERROR_READING_FILE = 31
+ ERROR_READING_FILE = 31,
+ FEATURE_DISABLED = 32
};
enum operations {
@@ -86,6 +87,8 @@ enum operations {
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
#define DOCKER_BINARY_KEY "docker.binary"
+#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
+#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
@@ -249,6 +252,12 @@ int check_dir(const char* npath, mode_t st_mode, mode_t desired,
int create_validate_dir(const char* npath, mode_t perm, const char* path,
int finalComponent);
+/** Check if tc (traffic control) support is enabled in configuration. */
+int is_tc_support_enabled();
+
+/** Check if docker support is enabled in configuration. */
+int is_docker_support_enabled();
+
/**
* Run a batch of tc commands that modify interface configuration
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1841b47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 222467a..5961e08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -43,26 +43,72 @@
#endif
static void display_usage(FILE *stream) {
- char *usage_template =
- "Usage: container-executor --checksetup\n" \
- " container-executor --mount-cgroups <hierarchy> <controller=path>...\n" \
- " container-executor --tc-modify-state <command-file>\n" \
- " container-executor --tc-read-state <command-file>\n" \
- " container-executor --tc-read-stats <command-file>\n" \
- " container-executor --run-docker <command-file>\n" \
- " container-executor <user> <yarn-user> <command> <command-args>\n" \
+ char usage_template[4096];
+
+ usage_template[0] = '\0';
+ strcat(usage_template,
+ "Usage: container-executor --checksetup\n"
+ " container-executor --mount-cgroups <hierarchy> "
+ "<controller=path>...\n" );
+
+ if(is_tc_support_enabled()) {
+ strcat(usage_template,
+ " container-executor --tc-modify-state <command-file>\n"
+ " container-executor --tc-read-state <command-file>\n"
+ " container-executor --tc-read-stats <command-file>\n" );
+ } else {
+ strcat(usage_template,
+ "[DISABLED] container-executor --tc-modify-state <command-file>\n"
+ "[DISABLED] container-executor --tc-read-state <command-file>\n"
+ "[DISABLED] container-executor --tc-read-stats <command-file>\n");
+ }
+
+ if(is_docker_support_enabled()) {
+ strcat(usage_template,
+ " container-executor --run-docker <command-file>\n");
+ } else {
+ strcat(usage_template,
+ "[DISABLED] container-executor --run-docker <command-file>\n");
+ }
+
+ strcat(usage_template,
+ " container-executor <user> <yarn-user> <command> <command-args>\n"
" where command and command-args: \n" \
- " initialize container: %2d appid tokens nm-local-dirs nm-log-dirs cmd app...\n" \
- " launch container: %2d appid containerid workdir container-script " \
- "tokens pidfile nm-local-dirs nm-log-dirs resources optional-tc-command-file\n" \
- " launch docker container: %2d appid containerid workdir container-script " \
- "tokens pidfile nm-local-dirs nm-log-dirs docker-command-file resources optional-tc-command-file\n" \
- " signal container: %2d container-pid signal\n" \
- " delete as user: %2d relative-path\n" ;
+ " initialize container: %2d appid tokens nm-local-dirs "
+ "nm-log-dirs cmd app...\n"
+ " launch container: %2d appid containerid workdir "
+ "container-script tokens pidfile nm-local-dirs nm-log-dirs resources ");
+
+ if(is_tc_support_enabled()) {
+ strcat(usage_template, "optional-tc-command-file\n");
+ } else {
+ strcat(usage_template, "\n");
+ }
+ if(is_docker_support_enabled()) {
+ strcat(usage_template,
+ " launch docker container: %2d appid containerid workdir "
+ "container-script tokens pidfile nm-local-dirs nm-log-dirs "
+ "docker-command-file resources ");
+ } else {
+ strcat(usage_template,
+ "[DISABLED] launch docker container: %2d appid containerid workdir "
+ "container-script tokens pidfile nm-local-dirs nm-log-dirs "
+ "docker-command-file resources ");
+ }
- fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER, LAUNCH_DOCKER_CONTAINER,
- SIGNAL_CONTAINER, DELETE_AS_USER);
+ if(is_tc_support_enabled()) {
+ strcat(usage_template, "optional-tc-command-file\n");
+ } else {
+ strcat(usage_template, "\n");
+ }
+
+ strcat(usage_template,
+ " signal container: %2d container-pid signal\n"
+ " delete as user: %2d relative-path\n");
+
+ fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER,
+ LAUNCH_DOCKER_CONTAINER, SIGNAL_CONTAINER, DELETE_AS_USER);
}
/* Sets up log files for normal/error logging */
@@ -143,6 +189,10 @@ static void assert_valid_setup(char *current_executable) {
}
+static void display_feature_disabled_message(const char* name) {
+ fprintf(ERRORFILE, "Feature disabled: %s\n", name);
+}
+
/* Use to store parsed input parmeters for various operations */
static struct {
char *cgroups_hierarchy;
@@ -199,47 +249,67 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
if (strcmp("--tc-modify-state", argv[1]) == 0) {
- if (argc != 3) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if(is_tc_support_enabled()) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_MODIFY_STATE;
+ return 0;
+ } else {
+ display_feature_disabled_message("traffic control");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.traffic_control_command_file = argv[optind++];
- *operation = TRAFFIC_CONTROL_MODIFY_STATE;
- return 0;
}
if (strcmp("--tc-read-state", argv[1]) == 0) {
- if (argc != 3) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if(is_tc_support_enabled()) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_READ_STATE;
+ return 0;
+ } else {
+ display_feature_disabled_message("traffic control");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.traffic_control_command_file = argv[optind++];
- *operation = TRAFFIC_CONTROL_READ_STATE;
- return 0;
}
if (strcmp("--tc-read-stats", argv[1]) == 0) {
- if (argc != 3) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if(is_tc_support_enabled()) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.traffic_control_command_file = argv[optind++];
+ *operation = TRAFFIC_CONTROL_READ_STATS;
+ return 0;
+ } else {
+ display_feature_disabled_message("traffic control");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.traffic_control_command_file = argv[optind++];
- *operation = TRAFFIC_CONTROL_READ_STATS;
- return 0;
}
if (strcmp("--run-docker", argv[1]) == 0) {
- if (argc != 3) {
- display_usage(stdout);
- return INVALID_ARGUMENT_NUMBER;
+ if(is_docker_support_enabled()) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ cmd_input.docker_command_file = argv[optind++];
+ *operation = RUN_DOCKER;
+ return 0;
+ } else {
+ display_feature_disabled_message("docker");
+ return FEATURE_DISABLED;
}
- optind++;
- cmd_input.docker_command_file = argv[optind++];
- *operation = RUN_DOCKER;
- return 0;
}
/* Now we have to validate 'run as user' operations that don't use
a 'long option' - we should fix this at some point. The validation/argument
@@ -285,51 +355,64 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
*operation = RUN_AS_USER_INITIALIZE_CONTAINER;
return 0;
case LAUNCH_DOCKER_CONTAINER:
- //kill me now.
- if (!(argc == 14 || argc == 15)) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for launch docker container\n",
- argc);
- fflush(ERRORFILE);
- return INVALID_ARGUMENT_NUMBER;
- }
+ if(is_docker_support_enabled()) {
+ //kill me now.
+ if (!(argc == 14 || argc == 15)) {
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for"
+ " launch docker container\n", argc);
+ fflush(ERRORFILE);
+ return INVALID_ARGUMENT_NUMBER;
+ }
- cmd_input.app_id = argv[optind++];
- cmd_input.container_id = argv[optind++];
- cmd_input.current_dir = argv[optind++];
- cmd_input.script_file = argv[optind++];
- cmd_input.cred_file = argv[optind++];
- cmd_input.pid_file = argv[optind++];
- cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list
- cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list
- cmd_input.docker_command_file = argv[optind++];
- resources = argv[optind++];// key,value pair describing resources
- resources_key = malloc(strlen(resources));
- resources_value = malloc(strlen(resources));
- if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
- get_kv_value(resources, resources_value, strlen(resources)) < 0) {
- fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
- resources);
- fflush(ERRORFILE);
- free(resources_key);
- free(resources_value);
- return INVALID_ARGUMENT_NUMBER;
- }
- //network isolation through tc
- if (argc == 15) {
- cmd_input.traffic_control_command_file = argv[optind++];
- }
+ cmd_input.app_id = argv[optind++];
+ cmd_input.container_id = argv[optind++];
+ cmd_input.current_dir = argv[optind++];
+ cmd_input.script_file = argv[optind++];
+ cmd_input.cred_file = argv[optind++];
+ cmd_input.pid_file = argv[optind++];
+ // good local dirs as a comma separated list
+ cmd_input.local_dirs = argv[optind++];
+ // good log dirs as a comma separated list
+ cmd_input.log_dirs = argv[optind++];
+ cmd_input.docker_command_file = argv[optind++];
+ // key,value pair describing resources
+ resources = argv[optind++];
+ resources_key = malloc(strlen(resources));
+ resources_value = malloc(strlen(resources));
+ if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
+ get_kv_value(resources, resources_value, strlen(resources)) < 0) {
+ fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
+ resources);
+ fflush(ERRORFILE);
+ free(resources_key);
+ free(resources_value);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ //network isolation through tc
+ if (argc == 15) {
+ if(is_tc_support_enabled()) {
+ cmd_input.traffic_control_command_file = argv[optind++];
+ } else {
+ display_feature_disabled_message("traffic control");
+ return FEATURE_DISABLED;
+ }
+ }
- cmd_input.resources_key = resources_key;
- cmd_input.resources_value = resources_value;
- cmd_input.resources_values = extract_values(resources_value);
- *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
- return 0;
+ cmd_input.resources_key = resources_key;
+ cmd_input.resources_value = resources_value;
+ cmd_input.resources_values = extract_values(resources_value);
+ *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
+ return 0;
+ } else {
+ display_feature_disabled_message("docker");
+ return FEATURE_DISABLED;
+ }
case LAUNCH_CONTAINER:
//kill me now.
if (!(argc == 13 || argc == 14)) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for launch container\n",
- argc);
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14)"
+ " for launch container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
@@ -358,7 +441,12 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
//network isolation through tc
if (argc == 14) {
- cmd_input.traffic_control_command_file = argv[optind++];
+ if(is_tc_support_enabled()) {
+ cmd_input.traffic_control_command_file = argv[optind++];
+ } else {
+ display_feature_disabled_message("traffic control");
+ return FEATURE_DISABLED;
+ }
}
cmd_input.resources_key = resources_key;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org