You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/10/09 06:54:33 UTC
hadoop git commit: YARN-4245. Generalize config file handling in
container-executor. Contributed by Sidharta Seethana.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 998e44a97 -> 50409404b
YARN-4245. Generalize config file handling in container-executor. Contributed by Sidharta Seethana.
(cherry picked from commit 8ed2e060e80c0def3fcb7604e0bd27c1c24d291e)
(cherry picked from commit 78919f8c341ec645cf9134991e3ae89a929b9184)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50409404
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50409404
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50409404
Branch: refs/heads/branch-2.8
Commit: 50409404bfa9a0c250529b16e60fe0b66bc3cd70
Parents: 998e44a
Author: Varun Vasudev <vv...@apache.org>
Authored: Mon Mar 7 16:18:35 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Sun Oct 9 11:53:03 2016 +0530
----------------------------------------------------------------------
.../container-executor/impl/configuration.c | 98 ++++++++++----------
.../container-executor/impl/configuration.h | 28 +++++-
.../impl/container-executor.c | 27 +++++-
.../impl/container-executor.h | 13 ++-
.../main/native/container-executor/impl/main.c | 4 +-
.../test/test-container-executor.c | 8 +-
6 files changed, 112 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 17cce75..3447524 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -34,34 +34,22 @@
#define MAX_SIZE 10
-struct confentry {
- const char *key;
- const char *value;
-};
-
-struct configuration {
- int size;
- struct confentry **confdetails;
-};
-
-struct configuration config={.size=0, .confdetails=NULL};
-
//clean up method for freeing configuration
-void free_configurations() {
+void free_configurations(struct configuration *cfg) {
int i = 0;
- for (i = 0; i < config.size; i++) {
- if (config.confdetails[i]->key != NULL) {
- free((void *)config.confdetails[i]->key);
+ for (i = 0; i < cfg->size; i++) {
+ if (cfg->confdetails[i]->key != NULL) {
+ free((void *)cfg->confdetails[i]->key);
}
- if (config.confdetails[i]->value != NULL) {
- free((void *)config.confdetails[i]->value);
+ if (cfg->confdetails[i]->value != NULL) {
+ free((void *)cfg->confdetails[i]->value);
}
- free(config.confdetails[i]);
+ free(cfg->confdetails[i]);
}
- if (config.size > 0) {
- free(config.confdetails);
+ if (cfg->size > 0) {
+ free(cfg->confdetails);
}
- config.size = 0;
+ cfg->size = 0;
}
/**
@@ -133,8 +121,8 @@ int check_configuration_permissions(const char* file_name) {
return 0;
}
-//function used to load the configurations present in the secure config
-void read_config(const char* file_name) {
+
+void read_config(const char* file_name, struct configuration *cfg) {
FILE *conf_file;
char *line;
char *equaltok;
@@ -152,9 +140,9 @@ void read_config(const char* file_name) {
#endif
//allocate space for ten configuration items.
- config.confdetails = (struct confentry **) malloc(sizeof(struct confentry *)
+ cfg->confdetails = (struct confentry **) malloc(sizeof(struct confentry *)
* MAX_SIZE);
- config.size = 0;
+ cfg->size = 0;
conf_file = fopen(file_name, "r");
if (conf_file == NULL) {
fprintf(ERRORFILE, "Invalid conf file provided : %s \n", file_name);
@@ -196,9 +184,9 @@ void read_config(const char* file_name) {
free(line);
continue;
}
- config.confdetails[config.size] = (struct confentry *) malloc(
+ cfg->confdetails[cfg->size] = (struct confentry *) malloc(
sizeof(struct confentry));
- if(config.confdetails[config.size] == NULL) {
+ if(cfg->confdetails[cfg->size] == NULL) {
fprintf(LOGFILE,
"Failed allocating memory for single configuration item\n");
goto cleanup;
@@ -208,10 +196,10 @@ void read_config(const char* file_name) {
fprintf(LOGFILE, "read_config : Adding conf key : %s \n", equaltok);
#endif
- memset(config.confdetails[config.size], 0, sizeof(struct confentry));
- config.confdetails[config.size]->key = (char *) malloc(
+ memset(cfg->confdetails[cfg->size], 0, sizeof(struct confentry));
+ cfg->confdetails[cfg->size]->key = (char *) malloc(
sizeof(char) * (strlen(equaltok)+1));
- strcpy((char *)config.confdetails[config.size]->key, equaltok);
+ strcpy((char *)cfg->confdetails[cfg->size]->key, equaltok);
equaltok = strtok_r(NULL, "=", &temp_equaltok);
if (equaltok == NULL) {
fprintf(LOGFILE, "configuration tokenization failed \n");
@@ -220,8 +208,8 @@ void read_config(const char* file_name) {
//means value is commented so don't store the key
if(equaltok[0] == '#') {
free(line);
- free((void *)config.confdetails[config.size]->key);
- free(config.confdetails[config.size]);
+ free((void *)cfg->confdetails[cfg->size]->key);
+ free(cfg->confdetails[cfg->size]);
continue;
}
@@ -229,27 +217,29 @@ void read_config(const char* file_name) {
fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok);
#endif
- config.confdetails[config.size]->value = (char *) malloc(
+ cfg->confdetails[cfg->size]->value = (char *) malloc(
sizeof(char) * (strlen(equaltok)+1));
- strcpy((char *)config.confdetails[config.size]->value, equaltok);
- if((config.size + 1) % MAX_SIZE == 0) {
- config.confdetails = (struct confentry **) realloc(config.confdetails,
- sizeof(struct confentry **) * (MAX_SIZE + config.size));
- if (config.confdetails == NULL) {
+ strcpy((char *)cfg->confdetails[cfg->size]->value, equaltok);
+ if((cfg->size + 1) % MAX_SIZE == 0) {
+ cfg->confdetails = (struct confentry **) realloc(cfg->confdetails,
+ sizeof(struct confentry **) * (MAX_SIZE + cfg->size));
+ if (cfg->confdetails == NULL) {
fprintf(LOGFILE,
"Failed re-allocating memory for configuration items\n");
goto cleanup;
}
}
- if(config.confdetails[config.size] )
- config.size++;
+ if(cfg->confdetails[cfg->size]) {
+ cfg->size++;
+ }
+
free(line);
}
//close the file
fclose(conf_file);
- if (config.size == 0) {
+ if (cfg->size == 0) {
fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_name);
exit(INVALID_CONFIG_FILE);
}
@@ -262,7 +252,7 @@ void read_config(const char* file_name) {
free(line);
}
fclose(conf_file);
- free_configurations();
+ free_configurations(cfg);
return;
}
@@ -272,11 +262,11 @@ void read_config(const char* file_name) {
* array, next time onwards used the populated array.
*
*/
-char * get_value(const char* key) {
+char * get_value(const char* key, struct configuration *cfg) {
int count;
- for (count = 0; count < config.size; count++) {
- if (strcmp(config.confdetails[count]->key, key) == 0) {
- return strdup(config.confdetails[count]->value);
+ for (count = 0; count < cfg->size; count++) {
+ if (strcmp(cfg->confdetails[count]->key, key) == 0) {
+ return strdup(cfg->confdetails[count]->value);
}
}
return NULL;
@@ -286,11 +276,21 @@ char * get_value(const char* key) {
* Function to return an array of values for a key.
* Value delimiter is assumed to be a ','.
*/
-char ** get_values(const char * key) {
- char *value = get_value(key);
+char ** get_values(const char * key, struct configuration *cfg) {
+ char *value = get_value(key, cfg);
return extract_values_delim(value, ",");
}
+/**
+ * Function to return an array of values for a key, using the specified
+ delimiter.
+ */
+char ** get_values_delim(const char * key, struct configuration *cfg,
+ const char *delim) {
+ char *value = get_value(key, cfg);
+ return extract_values_delim(value, delim);
+}
+
char ** extract_values_delim(char *value, const char *delim) {
char ** toPass = NULL;
char *tempTok = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index de5cc1d..8f87cb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -37,15 +37,33 @@ int check_configuration_permissions(const char* file_name);
*/
char *resolve_config_path(const char* file_name, const char *root);
-// read the given configuration file
-void read_config(const char* config_file);
+// Config data structures.
+struct confentry {
+ const char *key;
+ const char *value;
+};
+
+struct configuration {
+ int size;
+ struct confentry **confdetails;
+};
+
+// read the given configuration file into the specified config struct.
+void read_config(const char* config_file, struct configuration *cfg);
//method exposed to get the configurations
-char *get_value(const char* key);
+char *get_value(const char* key, struct configuration *cfg);
//function to return array of values pointing to the key. Values are
//comma seperated strings.
-char ** get_values(const char* key);
+char ** get_values(const char* key, struct configuration *cfg);
+
+/**
+ * Function to return an array of values for a key, using the specified
+ delimiter.
+ */
+char ** get_values_delim(const char * key, struct configuration *cfg,
+ const char *delim);
// Extracts array of values from the comma separated list of values.
char ** extract_values(char *value);
@@ -56,7 +74,7 @@ char ** extract_values_delim(char *value, const char *delim);
void free_values(char** values);
//method to free allocated configuration
-void free_configurations();
+void free_configurations(struct configuration *cfg);
/**
* If str is a string of the form key=val, find 'key'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 9df3b54..a10c8d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -60,6 +60,8 @@ FILE* ERRORFILE = NULL;
static uid_t nm_uid = -1;
static gid_t nm_gid = -1;
+struct configuration executor_cfg = {.size=0, .confdetails=NULL};
+
char *concatenate(char *concat_pattern, char *return_path_name,
int numArgs, ...);
@@ -68,6 +70,21 @@ void set_nm_uid(uid_t user, gid_t group) {
nm_gid = group;
}
+//function used to load the configurations present in the secure config
+void read_executor_config(const char* file_name) {
+ read_config(file_name, &executor_cfg);
+}
+
+//function used to free executor configuration data
+void free_executor_configurations() {
+ free_configurations(&executor_cfg);
+}
+
+//Lookup nodemanager group from container executor configuration.
+char *get_nodemanager_group() {
+ return get_value(NM_GROUP_KEY, &executor_cfg);
+}
+
/**
* get the executable filename.
*/
@@ -668,7 +685,7 @@ static struct passwd* get_user_info(const char* user) {
}
int is_whitelisted(const char *user) {
- char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY);
+ char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg);
char **users = whitelist;
if (whitelist != NULL) {
for(; *users; ++users) {
@@ -696,7 +713,7 @@ struct passwd* check_user(const char *user) {
fflush(LOGFILE);
return NULL;
}
- char *min_uid_str = get_value(MIN_USERID_KEY);
+ char *min_uid_str = get_value(MIN_USERID_KEY, &executor_cfg);
int min_uid = DEFAULT_MIN_USERID;
if (min_uid_str != NULL) {
char *end_ptr = NULL;
@@ -723,7 +740,7 @@ struct passwd* check_user(const char *user) {
free(user_info);
return NULL;
}
- char **banned_users = get_values(BANNED_USERS_KEY);
+ char **banned_users = get_values(BANNED_USERS_KEY, &executor_cfg);
banned_users = banned_users == NULL ?
(char**) DEFAULT_BANNED_USERS : banned_users;
char **banned_user = banned_users;
@@ -1072,7 +1089,7 @@ char* parse_docker_command_file(const char* command_file) {
int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_value(DOCKER_BINARY_KEY);
+ char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX);
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
char **args = extract_values_delim(docker_command_with_binary, " ");
@@ -1234,7 +1251,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
char buffer[BUFFER_SIZE];
char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_value(DOCKER_BINARY_KEY);
+ char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
if (docker_binary == NULL) {
docker_binary = "docker";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 4640bfb..fbab6bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -95,10 +95,15 @@ extern FILE *LOGFILE;
// the log file for error messages
extern FILE *ERRORFILE;
-
// get the executable's filename
char* get_executable();
+//function used to load the configurations present in the secure config
+void read_executor_config(const char* file_name);
+
+//Lookup nodemanager group from container executor configuration.
+char *get_nodemanager_group();
+
/**
* Check the permissions on the container-executor to make sure that security is
* permissible. For this, we need container-executor binary to
@@ -111,6 +116,12 @@ char* get_executable();
*/
int check_executor_permissions(char *executable_file);
+//function used to load the configurations present in the secure config.
+void read_executor_config(const char* file_name);
+
+//function used to free executor configuration data
+void free_executor_configurations();
+
// initialize the application directory
int initialize_app(const char *user, const char *app_id,
const char *credentials, char* const* local_dirs,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index ab45c7e..222467a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -112,11 +112,11 @@ static void assert_valid_setup(char *current_executable) {
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
- read_config(conf_file);
+ read_executor_config(conf_file);
free(conf_file);
// look up the node manager group in the config file
- char *nm_group = get_value(NM_GROUP_KEY);
+ char *nm_group = get_nodemanager_group();
if (nm_group == NULL) {
fprintf(ERRORFILE, "Can't get configured value for %s.\n", NM_GROUP_KEY);
flush_and_close_log_files();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50409404/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 839fdec..151c65b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -767,7 +767,7 @@ int main(int argc, char **argv) {
if (write_config_file(TEST_ROOT "/test.cfg", 1) != 0) {
exit(1);
}
- read_config(TEST_ROOT "/test.cfg");
+ read_executor_config(TEST_ROOT "/test.cfg");
local_dirs = extract_values(strdup(NM_LOCAL_DIRS));
log_dirs = extract_values(strdup(NM_LOG_DIRS));
@@ -839,14 +839,14 @@ int main(int argc, char **argv) {
seteuid(0);
// test_delete_user must run as root since that's how we use the delete_as_user
test_delete_user();
- free_configurations();
+ free_executor_configurations();
printf("\nTrying banned default user()\n");
if (write_config_file(TEST_ROOT "/test.cfg", 0) != 0) {
exit(1);
}
- read_config(TEST_ROOT "/test.cfg");
+ read_executor_config(TEST_ROOT "/test.cfg");
username = "bin";
test_check_user(1);
@@ -857,6 +857,6 @@ int main(int argc, char **argv) {
printf("\nFinished tests\n");
free(current_username);
- free_configurations();
+ free_executor_configurations();
return 0;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org