You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/08/16 00:28:00 UTC
[06/50] [abbrv] hadoop git commit: YARN-6033. Add support for
sections in container-executor configuration file. (Varun Vasudev via wandga)
YARN-6033. Add support for sections in container-executor configuration file. (Varun Vasudev via wandga)
Change-Id: Ibc6d2a959debe5d8ff2b51504149742449d1f1da
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec694145
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec694145
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec694145
Branch: refs/heads/YARN-6592
Commit: ec694145cf9c0ade7606813871ca2a4a371def8e
Parents: 63cfcb9
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Aug 9 10:51:29 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Aug 9 10:51:29 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-server-nodemanager/pom.xml | 38 ++
.../src/CMakeLists.txt | 22 +
.../container-executor/impl/configuration.c | 672 +++++++++++++------
.../container-executor/impl/configuration.h | 182 +++--
.../impl/container-executor.c | 39 +-
.../impl/container-executor.h | 52 +-
.../container-executor/impl/get_executable.c | 1 +
.../main/native/container-executor/impl/main.c | 17 +-
.../main/native/container-executor/impl/util.c | 134 ++++
.../main/native/container-executor/impl/util.h | 115 ++++
.../test-configurations/configuration-1.cfg | 31 +
.../test-configurations/configuration-2.cfg | 28 +
.../test/test-configurations/old-config.cfg | 25 +
.../test/test-container-executor.c | 15 +-
.../test/test_configuration.cc | 432 ++++++++++++
.../native/container-executor/test/test_main.cc | 29 +
.../native/container-executor/test/test_util.cc | 138 ++++
17 files changed, 1649 insertions(+), 321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 28ee0d9..a50a769 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -215,6 +215,44 @@
<results>${project.build.directory}/native-results</results>
</configuration>
</execution>
+ <execution>
+ <id>cetest</id>
+ <goals><goal>cmake-test</goal></goals>
+ <phase>test</phase>
+ <configuration>
+ <!-- this should match the xml name without the TEST-part down below -->
+ <testName>cetest</testName>
+ <workingDirectory>${project.build.directory}/native/test</workingDirectory>
+ <source>${basedir}/src</source>
+ <binary>${project.build.directory}/native/test/cetest</binary>
+ <args>
+ <arg>--gtest_filter=-Perf.</arg>
+ <arg>--gtest_output=xml:${project.build.directory}/surefire-reports/TEST-cetest.xml</arg>
+ </args>
+ <results>${project.build.directory}/surefire-reports</results>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>make</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy todir="${project.build.directory}/native/test/"
+ overwrite="true">
+ <fileset dir="${basedir}/src/main/native/container-executor/resources/test" />
+ </copy>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 5b52536..100d7ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -19,6 +19,9 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common)
include(HadoopCommon)
+# Set gtest path
+set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common/src/main/native/gtest)
+
# determine if container-executor.conf.dir is an absolute
# path in case the OS we're compiling on doesn't have
# a hook in get_executable. We'll use this define
@@ -80,12 +83,20 @@ endfunction()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}
+ ${GTEST_SRC_DIR}/include
main/native/container-executor
main/native/container-executor/impl
)
+# add gtest as system library to suppress gcc warnings
+include_directories(SYSTEM ${GTEST_SRC_DIR}/include)
+
configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc)
+set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
+
add_library(container
+ main/native/container-executor/impl/util.c
main/native/container-executor/impl/configuration.c
main/native/container-executor/impl/container-executor.c
main/native/container-executor/impl/get_executable.c
@@ -95,9 +106,11 @@ add_library(container
add_executable(container-executor
main/native/container-executor/impl/main.c
)
+
target_link_libraries(container-executor
container
)
+
output_directory(container-executor target/usr/local/bin)
add_executable(test-container-executor
@@ -107,3 +120,12 @@ target_link_libraries(test-container-executor
container ${EXTRA_LIBS}
)
output_directory(test-container-executor target/usr/local/bin)
+
+# unit tests for container executor
+add_executable(cetest
+ main/native/container-executor/impl/util.c
+ main/native/container-executor/test/test_configuration.cc
+ main/native/container-executor/test/test_main.cc
+ main/native/container-executor/test/test_util.cc)
+target_link_libraries(cetest gtest)
+output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index a6d7a9c..12dbc4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -20,35 +20,55 @@
#include <libgen.h>
#include "configuration.h"
-#include "container-executor.h"
+#include "util.h"
+#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <errno.h>
#include <unistd.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
-#include <sys/types.h>
-#include <limits.h>
-#include <ctype.h>
#define MAX_SIZE 10
+static const char COMMENT_BEGIN_CHAR = '#';
+static const char SECTION_LINE_BEGIN_CHAR = '[';
+static const char SECTION_LINE_END_CHAR = ']';
+
+//clean up method for freeing section
+void free_section(struct section *section) {
+ int i = 0;
+ for (i = 0; i < section->size; i++) {
+ if (section->kv_pairs[i]->key != NULL) {
+ free((void *) section->kv_pairs[i]->key);
+ }
+ if (section->kv_pairs[i]->value != NULL) {
+ free((void *) section->kv_pairs[i]->value);
+ }
+ free(section->kv_pairs[i]);
+ }
+ if (section->kv_pairs) {
+ free(section->kv_pairs);
+ section->kv_pairs = NULL;
+ }
+ if (section->name) {
+ free(section->name);
+ section->name = NULL;
+ }
+ section->size = 0;
+}
+
//clean up method for freeing configuration
-void free_configurations(struct configuration *cfg) {
+void free_configuration(struct configuration *cfg) {
int i = 0;
for (i = 0; i < cfg->size; i++) {
- if (cfg->confdetails[i]->key != NULL) {
- free((void *)cfg->confdetails[i]->key);
+ if (cfg->sections[i] != NULL) {
+ free_section(cfg->sections[i]);
}
- if (cfg->confdetails[i]->value != NULL) {
- free((void *)cfg->confdetails[i]->value);
- }
- free(cfg->confdetails[i]);
}
- if (cfg->size > 0) {
- free(cfg->confdetails);
+ if (cfg->sections) {
+ free(cfg->sections);
}
cfg->size = 0;
}
@@ -65,13 +85,13 @@ static int is_only_root_writable(const char *file) {
}
if (file_stat.st_uid != 0) {
fprintf(ERRORFILE, "File %s must be owned by root, but is owned by %" PRId64 "\n",
- file, (int64_t)file_stat.st_uid);
+ file, (int64_t) file_stat.st_uid);
return 0;
}
if ((file_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) {
fprintf(ERRORFILE,
- "File %s must not be world or group writable, but is %03lo\n",
- file, (unsigned long)file_stat.st_mode & (~S_IFMT));
+ "File %s must not be world or group writable, but is %03lo\n",
+ file, (unsigned long) file_stat.st_mode & (~S_IFMT));
return 0;
}
return 1;
@@ -82,9 +102,9 @@ static int is_only_root_writable(const char *file) {
*
* NOTE: relative path names are resolved relative to the second argument not getwd(3)
*/
-char *resolve_config_path(const char* file_name, const char *root) {
+char *resolve_config_path(const char *file_name, const char *root) {
const char *real_fname = NULL;
- char buffer[EXECUTOR_PATH_MAX*2 + 1];
+ char buffer[EXECUTOR_PATH_MAX * 2 + 1];
if (file_name[0] == '/') {
real_fname = file_name;
@@ -96,7 +116,7 @@ char *resolve_config_path(const char* file_name, const char *root) {
#ifdef HAVE_CANONICALIZE_FILE_NAME
char * ret = (real_fname == NULL) ? NULL : canonicalize_file_name(real_fname);
#else
- char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
+ char *ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
#endif
#ifdef DEBUG
fprintf(stderr,"ret = %s\n", ret);
@@ -112,10 +132,19 @@ char *resolve_config_path(const char* file_name, const char *root) {
* configuration and potentially cause damage.
* returns 0 if permissions are ok
*/
-int check_configuration_permissions(const char* file_name) {
+int check_configuration_permissions(const char *file_name) {
+ if (!file_name) {
+ return -1;
+ }
+
// copy the input so that we can modify it with dirname
- char* dir = strdup(file_name);
- char* buffer = dir;
+ char *dir = strdup(file_name);
+ if (!dir) {
+ fprintf(stderr, "Failed to make a copy of filename in %s.\n", __func__);
+ return -1;
+ }
+
+ char *buffer = dir;
do {
if (!is_only_root_writable(dir)) {
free(buffer);
@@ -128,167 +157,396 @@ int check_configuration_permissions(const char* file_name) {
}
/**
- * Trim whitespace from beginning and end.
-*/
-char* trim(char* input)
-{
- char *val_begin;
- char *val_end;
- char *ret;
-
- if (input == NULL) {
- return NULL;
+ * Read a line from the the config file and return it without the newline.
+ * The caller must free the memory allocated.
+ */
+static char *read_config_line(FILE *conf_file) {
+ char *line = NULL;
+ size_t linesize = 100000;
+ ssize_t size_read = 0;
+ size_t eol = 0;
+
+ line = (char *) malloc(linesize);
+ if (line == NULL) {
+ fprintf(ERRORFILE, "malloc failed while reading configuration file.\n");
+ exit(OUT_OF_MEMORY);
+ }
+ size_read = getline(&line, &linesize, conf_file);
+
+ //feof returns true only after we read past EOF.
+ //so a file with no new line, at last can reach this place
+ //if size_read returns negative check for eof condition
+ if (size_read == -1) {
+ free(line);
+ line = NULL;
+ if (!feof(conf_file)) {
+ fprintf(ERRORFILE, "Line read returned -1 without eof\n");
+ exit(INVALID_CONFIG_FILE);
+ }
+ } else {
+ eol = strlen(line) - 1;
+ if (line[eol] == '\n') {
+ //trim the ending new line
+ line[eol] = '\0';
}
+ }
+ return line;
+}
- val_begin = input;
- val_end = input + strlen(input);
+/**
+ * Return if the given line is a comment line.
+ *
+ * @param line the line to check
+ *
+ * @return 1 if the line is a comment line, 0 otherwise
+ */
+static int is_comment_line(const char *line) {
+ if (line != NULL) {
+ return (line[0] == COMMENT_BEGIN_CHAR);
+ }
+ return 0;
+}
- while (val_begin < val_end && isspace(*val_begin))
- val_begin++;
- while (val_end > val_begin && isspace(*(val_end - 1)))
- val_end--;
+/**
+ * Return if the given line is a section start line.
+ *
+ * @param line the line to check
+ *
+ * @return 1 if the line is a section start line, 0 otherwise
+ */
+static int is_section_start_line(const char *line) {
+ size_t len = 0;
+ if (line != NULL) {
+ len = strlen(line) - 1;
+ return (line[0] == SECTION_LINE_BEGIN_CHAR
+ && line[len] == SECTION_LINE_END_CHAR);
+ }
+ return 0;
+}
- ret = (char *) malloc(
- sizeof(char) * (val_end - val_begin + 1));
- if (ret == NULL) {
- fprintf(ERRORFILE, "Allocation error\n");
+/**
+ * Return the name of the section from the given section start line. The
+ * caller must free the memory used.
+ *
+ * @param line the line to extract the section name from
+ *
+ * @return string with the name of the section, NULL otherwise
+ */
+static char *get_section_name(const char *line) {
+ char *name = NULL;
+ size_t len;
+
+ if (is_section_start_line(line)) {
+ // length of the name is the line - 2(to account for '[' and ']')
+ len = strlen(line) - 2;
+ name = (char *) malloc(len + 1);
+ if (name == NULL) {
+ fprintf(ERRORFILE, "malloc failed while reading section name.\n");
exit(OUT_OF_MEMORY);
}
-
- strncpy(ret, val_begin, val_end - val_begin);
- ret[val_end - val_begin] = '\0';
- return ret;
+ strncpy(name, line + sizeof(char), len);
+ name[len] = '\0';
+ }
+ return name;
}
-void read_config(const char* file_name, struct configuration *cfg) {
- FILE *conf_file;
- char *line;
+/**
+ * Read an entry for the section from the line. Function returns 0 if an entry
+ * was found, non-zero otherwise. Return values less than 0 indicate an error
+ * with the config file.
+ *
+ * @param line the line to read the entry from
+ * @param section the struct to read the entry into
+ *
+ * @return 0 if an entry was found
+ * <0 for config file errors
+ * >0 for issues such as empty line
+ *
+ */
+static int read_section_entry(const char *line, struct section *section) {
char *equaltok;
char *temp_equaltok;
- size_t linesize = 1000;
- int size_read = 0;
-
- if (file_name == NULL) {
- fprintf(ERRORFILE, "Null configuration filename passed in\n");
- exit(INVALID_CONFIG_FILE);
+ const char *splitter = "=";
+ char *buffer;
+ size_t len = 0;
+ if (line == NULL || section == NULL) {
+ fprintf(ERRORFILE, "NULL params passed to read_section_entry");
+ return -1;
+ }
+ len = strlen(line);
+ if (len == 0) {
+ return 1;
+ }
+ if ((section->size) % MAX_SIZE == 0) {
+ section->kv_pairs = (struct kv_pair **) realloc(
+ section->kv_pairs,
+ sizeof(struct kv_pair *) * (MAX_SIZE + section->size));
+ if (section->kv_pairs == NULL) {
+ fprintf(ERRORFILE,
+ "Failed re-allocating memory for configuration items\n");
+ exit(OUT_OF_MEMORY);
+ }
}
- #ifdef DEBUG
- fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_name);
- #endif
+ buffer = strdup(line);
+ if (!buffer) {
+ fprintf(ERRORFILE, "Failed to allocating memory for line, %s\n", __func__);
+ exit(OUT_OF_MEMORY);
+ }
- //allocate space for ten configuration items.
- cfg->confdetails = (struct confentry **) malloc(sizeof(struct confentry *)
- * MAX_SIZE);
- cfg->size = 0;
- conf_file = fopen(file_name, "r");
- if (conf_file == NULL) {
- fprintf(ERRORFILE, "Invalid conf file provided : %s \n", file_name);
+ //tokenize first to get key and list of values.
+ //if no equals is found ignore this line, can be an empty line also
+ equaltok = strtok_r(buffer, splitter, &temp_equaltok);
+ if (equaltok == NULL) {
+ fprintf(ERRORFILE, "Error with line '%s', no '=' found\n", buffer);
exit(INVALID_CONFIG_FILE);
}
- while(!feof(conf_file)) {
- line = (char *) malloc(linesize);
- if(line == NULL) {
- fprintf(ERRORFILE, "malloc failed while reading configuration file.\n");
- exit(OUT_OF_MEMORY);
+ section->kv_pairs[section->size] = (struct kv_pair *) malloc(
+ sizeof(struct kv_pair));
+ if (section->kv_pairs[section->size] == NULL) {
+ fprintf(ERRORFILE, "Failed allocating memory for single section item\n");
+ exit(OUT_OF_MEMORY);
+ }
+ memset(section->kv_pairs[section->size], 0,
+ sizeof(struct kv_pair));
+ section->kv_pairs[section->size]->key = trim(equaltok);
+
+ equaltok = strtok_r(NULL, splitter, &temp_equaltok);
+ if (equaltok == NULL) {
+ // this can happen because no value was set
+ // e.g. banned.users=#this is a comment
+ int has_values = 1;
+ if (strstr(line, splitter) == NULL) {
+ fprintf(ERRORFILE, "configuration tokenization failed, error with line %s\n", line);
+ has_values = 0;
}
- size_read = getline(&line,&linesize,conf_file);
- //feof returns true only after we read past EOF.
- //so a file with no new line, at last can reach this place
- //if size_read returns negative check for eof condition
- if (size_read == -1) {
- free(line);
- if(!feof(conf_file)){
- exit(INVALID_CONFIG_FILE);
- } else {
- break;
- }
- }
- int eol = strlen(line) - 1;
- if(line[eol] == '\n') {
- //trim the ending new line
- line[eol] = '\0';
+ // It is not a valid line, free memory.
+ free((void *) section->kv_pairs[section->size]->key);
+ free((void *) section->kv_pairs[section->size]);
+ section->kv_pairs[section->size] = NULL;
+ free(buffer);
+
+ // Return -1 when no values
+ if (!has_values) {
+ return -1;
}
- //comment line
- if(line[0] == '#') {
- free(line);
- continue;
+
+ // Return 2 for comments
+ return 2;
+ }
+
+#ifdef DEBUG
+ fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok);
+#endif
+
+ section->kv_pairs[section->size]->value = trim(equaltok);
+ section->size++;
+ free(buffer);
+ return 0;
+}
+
+/**
+ * Remove any trailing comment from the supplied line. Function modifies the
+ * argument provided.
+ *
+ * @param line the line from which to remove the comment
+ */
+static void trim_comment(char *line) {
+ char *begin_comment = NULL;
+ if (line != NULL) {
+ begin_comment = strchr(line, COMMENT_BEGIN_CHAR);
+ if (begin_comment != NULL) {
+ *begin_comment = '\0';
}
- //tokenize first to get key and list of values.
- //if no equals is found ignore this line, can be an empty line also
- equaltok = strtok_r(line, "=", &temp_equaltok);
- if(equaltok == NULL) {
+ }
+}
+
+/**
+ * Allocate a section struct and initialize it. The memory must be freed by
+ * the caller. Function calls exit if any error occurs.
+ *
+ * @return pointer to the allocated section struct
+ *
+ */
+static struct section *allocate_section() {
+ struct section *section = (struct section *) malloc(sizeof(struct section));
+ if (section == NULL) {
+ fprintf(ERRORFILE, "malloc failed while allocating section.\n");
+ exit(OUT_OF_MEMORY);
+ }
+ section->name = NULL;
+ section->kv_pairs = NULL;
+ section->size = 0;
+ return section;
+}
+
+/**
+ * Populate the given section struct with fields from the config file.
+ *
+ * @param conf_file the file to read from
+ * @param section pointer to the section struct to populate
+ *
+ */
+static void populate_section_fields(FILE *conf_file, struct section *section) {
+ char *line;
+ long int offset = 0;
+ while (!feof(conf_file)) {
+ offset = ftell(conf_file);
+ line = read_config_line(conf_file);
+ if (line != NULL) {
+ if (!is_comment_line(line)) {
+ trim_comment(line);
+ if (!is_section_start_line(line)) {
+ if (section->name != NULL) {
+ if (read_section_entry(line, section) < 0) {
+ fprintf(ERRORFILE, "Error parsing line %s", line);
+ exit(INVALID_CONFIG_FILE);
+ }
+ } else {
+ fprintf(ERRORFILE, "Line '%s' doesn't belong to a section\n",
+ line);
+ exit(INVALID_CONFIG_FILE);
+ }
+ } else {
+ if (section->name == NULL) {
+ section->name = get_section_name(line);
+ if (strlen(section->name) == 0) {
+ fprintf(ERRORFILE, "Empty section name");
+ exit(INVALID_CONFIG_FILE);
+ }
+ } else {
+ // we've reached the next section
+ fseek(conf_file, offset, SEEK_SET);
+ free(line);
+ return;
+ }
+ }
+ }
free(line);
- continue;
- }
- cfg->confdetails[cfg->size] = (struct confentry *) malloc(
- sizeof(struct confentry));
- if(cfg->confdetails[cfg->size] == NULL) {
- fprintf(LOGFILE,
- "Failed allocating memory for single configuration item\n");
- goto cleanup;
}
+ }
+}
- #ifdef DEBUG
- fprintf(LOGFILE, "read_config : Adding conf key : %s \n", equaltok);
- #endif
+/**
+ * Read the section current section from the conf file. Section start is
+ * marked by lines of the form '[section-name]' and continue till the next
+ * section.
+ */
+static struct section *read_section(FILE *conf_file) {
+ struct section *section = allocate_section();
+ populate_section_fields(conf_file, section);
+ if (section->name == NULL) {
+ free_section(section);
+ section = NULL;
+ }
+ return section;
+}
+
+/**
+ * Merge two sections and free the second one after the merge, if desired.
+ * @param section1 the first section
+ * @param section2 the second section
+ * @param free_second_section free the second section if set
+ */
+static void merge_sections(struct section *section1, struct section *section2, const int free_second_section) {
+ int i = 0;
+ section1->kv_pairs = (struct kv_pair **) realloc(
+ section1->kv_pairs,
+ sizeof(struct kv_pair *) * (section1->size + section2->size));
+ if (section1->kv_pairs == NULL) {
+ fprintf(ERRORFILE,
+ "Failed re-allocating memory for configuration items\n");
+ exit(OUT_OF_MEMORY);
+ }
+ for (i = 0; i < section2->size; ++i) {
+ section1->kv_pairs[section1->size + i] = section2->kv_pairs[i];
+ }
+ section1->size += section2->size;
+ if (free_second_section) {
+ free(section2->name);
+ memset(section2, 0, sizeof(*section2));
+ free(section2);
+ }
+}
- memset(cfg->confdetails[cfg->size], 0, sizeof(struct confentry));
- cfg->confdetails[cfg->size]->key = trim(equaltok);
+int read_config(const char *file_path, struct configuration *cfg) {
+ FILE *conf_file;
- equaltok = strtok_r(NULL, "=", &temp_equaltok);
- if (equaltok == NULL) {
- fprintf(LOGFILE, "configuration tokenization failed \n");
- goto cleanup;
- }
- //means value is commented so don't store the key
- if(equaltok[0] == '#') {
- free(line);
- free((void *)cfg->confdetails[cfg->size]->key);
- free(cfg->confdetails[cfg->size]);
- continue;
+ if (file_path == NULL) {
+ fprintf(ERRORFILE, "Null configuration filename passed in\n");
+ return INVALID_CONFIG_FILE;
+ }
+
+#ifdef DEBUG
+ fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_path);
+#endif
+
+ cfg->size = 0;
+ conf_file = fopen(file_path, "r");
+ if (conf_file == NULL) {
+ fprintf(ERRORFILE, "Invalid conf file provided, unable to open file"
+ " : %s \n", file_path);
+ return (INVALID_CONFIG_FILE);
+ }
+
+ cfg->sections = (struct section **) malloc(
+ sizeof(struct section *) * MAX_SIZE);
+ if (!cfg->sections) {
+ fprintf(ERRORFILE,
+ "Failed to allocate memory for configuration sections\n");
+ exit(OUT_OF_MEMORY);
+ }
+
+ // populate any entries in the older format(no sections)
+ cfg->sections[cfg->size] = allocate_section();
+ cfg->sections[cfg->size]->name = strdup("");
+ populate_section_fields(conf_file, cfg->sections[cfg->size]);
+ if (cfg->sections[cfg->size]) {
+ if (cfg->sections[cfg->size]->size) {
+ cfg->size++;
+ } else {
+ free_section(cfg->sections[cfg->size]);
}
+ }
- #ifdef DEBUG
- fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok);
- #endif
-
- cfg->confdetails[cfg->size]->value = trim(equaltok);
- if((cfg->size + 1) % MAX_SIZE == 0) {
- cfg->confdetails = (struct confentry **) realloc(cfg->confdetails,
- sizeof(struct confentry **) * (MAX_SIZE + cfg->size));
- if (cfg->confdetails == NULL) {
- fprintf(LOGFILE,
- "Failed re-allocating memory for configuration items\n");
- goto cleanup;
+ // populate entries in the sections format
+ while (!feof(conf_file)) {
+ cfg->sections[cfg->size] = NULL;
+ struct section *new_section = read_section(conf_file);
+ if (new_section != NULL) {
+ struct section *existing_section =
+ get_configuration_section(new_section->name, cfg);
+ if (existing_section != NULL) {
+ merge_sections((struct section *) existing_section, new_section, 1);
+ } else {
+ cfg->sections[cfg->size] = new_section;
}
}
- if(cfg->confdetails[cfg->size]) {
- cfg->size++;
- }
- free(line);
+ // Check if we need to expand memory for sections.
+ if (cfg->sections[cfg->size]) {
+ if ((cfg->size + 1) % MAX_SIZE == 0) {
+ cfg->sections = (struct section **) realloc(cfg->sections,
+ sizeof(struct sections *) * (MAX_SIZE + cfg->size));
+ if (cfg->sections == NULL) {
+ fprintf(ERRORFILE,
+ "Failed re-allocating memory for configuration items\n");
+ exit(OUT_OF_MEMORY);
+ }
+ }
+ cfg->size++;
+ }
}
- //close the file
fclose(conf_file);
if (cfg->size == 0) {
- fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_name);
- exit(INVALID_CONFIG_FILE);
- }
-
- //clean up allocated file name
- return;
- //free spaces alloced.
- cleanup:
- if (line != NULL) {
- free(line);
+ free_configuration(cfg);
+ fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_path);
+ return INVALID_CONFIG_FILE;
}
- fclose(conf_file);
- free_configurations(cfg);
- return;
+ return 0;
}
/*
@@ -297,11 +555,14 @@ void read_config(const char* file_name, struct configuration *cfg) {
* array, next time onwards used the populated array.
*
*/
-char * get_value(const char* key, struct configuration *cfg) {
+char *get_section_value(const char *key, const struct section *section) {
int count;
- for (count = 0; count < cfg->size; count++) {
- if (strcmp(cfg->confdetails[count]->key, key) == 0) {
- return strdup(cfg->confdetails[count]->value);
+ if (key == NULL || section == NULL) {
+ return NULL;
+ }
+ for (count = 0; count < section->size; count++) {
+ if (strcmp(section->kv_pairs[count]->key, key) == 0) {
+ return strdup(section->kv_pairs[count]->value);
}
}
return NULL;
@@ -311,61 +572,80 @@ char * get_value(const char* key, struct configuration *cfg) {
* Function to return an array of values for a key.
* Value delimiter is assumed to be a ','.
*/
-char ** get_values(const char * key, struct configuration *cfg) {
- char *value = get_value(key, cfg);
- return extract_values_delim(value, ",");
+char **get_section_values(const char *key, const struct section *cfg) {
+ return get_section_values_delimiter(key, cfg, ",");
}
/**
* Function to return an array of values for a key, using the specified
delimiter.
*/
-char ** get_values_delim(const char * key, struct configuration *cfg,
- const char *delim) {
- char *value = get_value(key, cfg);
- return extract_values_delim(value, delim);
+char **get_section_values_delimiter(const char *key, const struct section *cfg,
+ const char *delim) {
+ if (key == NULL || cfg == NULL || delim == NULL) {
+ return NULL;
+ }
+ char *value = get_section_value(key, cfg);
+ char **split_values = split_delimiter(value, delim);
+
+ if (value) {
+ free(value);
+ }
+
+ return split_values;
}
-char ** extract_values_delim(char *value, const char *delim) {
- char ** toPass = NULL;
- char *tempTok = NULL;
- char *tempstr = NULL;
- int size = 0;
- int toPassSize = MAX_SIZE;
- //first allocate any array of 10
- if(value != NULL) {
- toPass = (char **) malloc(sizeof(char *) * toPassSize);
- tempTok = strtok_r((char *)value, delim, &tempstr);
- while (tempTok != NULL) {
- toPass[size++] = tempTok;
- if(size == toPassSize) {
- toPassSize += MAX_SIZE;
- toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
- }
- tempTok = strtok_r(NULL, delim, &tempstr);
- }
+char *get_configuration_value(const char *key, const char *section,
+ const struct configuration *cfg) {
+ const struct section *section_ptr;
+ if (key == NULL || section == NULL || cfg == NULL) {
+ return NULL;
}
- if (toPass != NULL) {
- toPass[size] = NULL;
+ section_ptr = get_configuration_section(section, cfg);
+ if (section_ptr != NULL) {
+ return get_section_value(key, section_ptr);
}
- return toPass;
+ return NULL;
}
-/**
- * Extracts array of values from the '%' separated list of values.
- */
-char ** extract_values(char *value) {
- return extract_values_delim(value, "%");
+char **get_configuration_values(const char *key, const char *section,
+ const struct configuration *cfg) {
+ const struct section *section_ptr;
+ if (key == NULL || section == NULL || cfg == NULL) {
+ return NULL;
+ }
+ section_ptr = get_configuration_section(section, cfg);
+ if (section_ptr != NULL) {
+ return get_section_values(key, section_ptr);
+ }
+ return NULL;
+}
+
+char **get_configuration_values_delimiter(const char *key, const char *section,
+ const struct configuration *cfg, const char *delim) {
+ const struct section *section_ptr;
+ if (key == NULL || section == NULL || cfg == NULL || delim == NULL) {
+ return NULL;
+ }
+ section_ptr = get_configuration_section(section, cfg);
+ if (section_ptr != NULL) {
+ return get_section_values_delimiter(key, section_ptr, delim);
+ }
+ return NULL;
}
-// free an entry set of values
-void free_values(char** values) {
- if (*values != NULL) {
- free(*values);
+struct section *get_configuration_section(const char *section,
+ const struct configuration *cfg) {
+ int i = 0;
+ if (cfg == NULL || section == NULL) {
+ return NULL;
}
- if (values != NULL) {
- free(values);
+ for (i = 0; i < cfg->size; ++i) {
+ if (strcmp(cfg->sections[i]->name, section) == 0) {
+ return cfg->sections[i];
+ }
}
+ return NULL;
}
/**
@@ -376,12 +656,12 @@ int get_kv_key(const char *input, char *out, size_t out_len) {
if (input == NULL)
return -EINVAL;
- char *split = strchr(input, '=');
+ const char *split = strchr(input, '=');
if (split == NULL)
return -EINVAL;
- int key_len = split - input;
+ unsigned long key_len = split - input;
if (out_len < (key_len + 1) || out == NULL)
return -ENAMETOOLONG;
@@ -400,13 +680,13 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
if (input == NULL)
return -EINVAL;
- char *split = strchr(input, '=');
+ const char *split = strchr(input, '=');
if (split == NULL)
return -EINVAL;
split++; // advance past '=' to the value
- int val_len = (input + strlen(input)) - split;
+ unsigned long val_len = (input + strlen(input)) - split;
if (out_len < (val_len + 1) || out == NULL)
return -ENAMETOOLONG;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 2d14867..1ea5561 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -16,6 +16,9 @@
* limitations under the License.
*/
+#ifndef __YARN_CONTAINER_EXECUTOR_CONFIG_H__
+#define __YARN_CONTAINER_EXECUTOR_CONFIG_H__
+
#ifdef __FreeBSD__
#define _WITH_GETLINE
#endif
@@ -23,62 +26,160 @@
#include <stddef.h>
/** Define a platform-independent constant instead of using PATH_MAX */
-
#define EXECUTOR_PATH_MAX 4096
-/**
- * Ensure that the configuration file and all of the containing directories
- * are only writable by root. Otherwise, an attacker can change the
- * configuration and potentially cause damage.
- * returns 0 if permissions are ok
- */
-int check_configuration_permissions(const char* file_name);
-
-/**
- * Return a string with the configuration file path name resolved via realpath(3)
- *
- * NOTE: relative path names are resolved relative to the second argument not getwd(3)
- */
-char *resolve_config_path(const char* file_name, const char *root);
-
-// Config data structures.
-struct confentry {
+// Configuration data structures.
+struct kv_pair {
const char *key;
const char *value;
};
+struct section {
+ int size;
+ char *name;
+ struct kv_pair **kv_pairs;
+};
+
struct configuration {
int size;
- struct confentry **confdetails;
+ struct section **sections;
};
-// read the given configuration file into the specified config struct.
-void read_config(const char* config_file, struct configuration *cfg);
+/**
+ * Function to ensure that the configuration file and all of the containing
+ * directories are only writable by root. Otherwise, an attacker can change
+ * the configuration and potentially cause damage.
+ *
+ * @param file_name name of the config file
+ *
+ * @returns 0 if permissions are correct, non-zero on error
+ */
+int check_configuration_permissions(const char *file_name);
+
+/**
+ * Return a string with the configuration file path name resolved via
+ * realpath(3). Relative path names are resolved relative to the second
+ * argument and not getwd(3). It's up to the caller to free the returned
+ * value.
+ *
+ * @param file_name name of the config file
+ * @param root the path against which relative path names are to be resolved
+ *
+ * @returns the resolved configuration file path
+ */
+char* resolve_config_path(const char *file_name, const char *root);
-//method exposed to get the configurations
-char *get_value(const char* key, struct configuration *cfg);
+/**
+ * Read the given configuration file into the specified configuration struct.
+ * It's the responsibility of the caller to call free_configurations to free
+ * the allocated memory. The function will check to ensure that the
+ * configuration file has the appropriate owner and permissions.
+ *
+ * @param file_path name of the configuration file to be read
+ * @param cfg the configuration structure to be filled.
+ *
+ * @return 0 on success, non-zero if there was an error
+ */
+int read_config(const char *file_path, struct configuration *cfg);
+
+/**
+ * Get the value for a key in the specified section. It's up to the caller to
+ * free the memory used for storing the return value.
+ *
+ * @param key key the name of the key
+ * @param section the section to be looked up
+ *
+ * @return pointer to the value if the key was found, null otherwise
+ */
+char* get_section_value(const char *key, const struct section *section);
-//function to return array of values pointing to the key. Values are
-//comma seperated strings.
-char ** get_values(const char* key, struct configuration *cfg);
+/**
+ * Function to get the values for a given key in the specified section.
+ * The value is split by ",". It's up to the caller to free the memory used
+ * for storing the return values.
+ *
+ * @param key the key to be looked up
+ * @param section the section to be looked up
+ *
+ * @return array of values, null if the key was not found
+ */
+char** get_section_values(const char *key, const struct section *section);
/**
- * Function to return an array of values for a key, using the specified
- delimiter.
+ * Function to get the values for a given key in the specified section.
+ * The value is split by the specified delimiter. It's up to the caller to
+ * free the memory used for storing the return values.
+ *
+ * @param key the key to be looked up
+ * @param section the section to be looked up
+ * @param delimiter the delimiter to be used to split the value
+ *
+ * @return array of values, null if the key was not found
*/
-char ** get_values_delim(const char * key, struct configuration *cfg,
+char** get_section_values_delimiter(const char *key, const struct section *section,
const char *delim);
-// Extracts array of values from the comma separated list of values.
-char ** extract_values(char *value);
+/**
+ * Get the value for a key in the specified section in the specified
+ * configuration. It's up to the caller to free the memory used for storing
+ * the return value.
+ *
+ * @param key key the name of the key
+ * @param section the name section to be looked up
+ * @param cfg the configuration to be used
+ *
+ * @return pointer to the value if the key was found, null otherwise
+ */
+char* get_configuration_value(const char *key, const char* section,
+ const struct configuration *cfg);
+
+/**
+ * Function to get the values for a given key in the specified section in the
+ * specified configuration. The value is split by ",". It's up to the caller to
+ * free the memory used for storing the return values.
+ *
+ * @param key the key to be looked up
+ * @param section the name of the section to be looked up
+ * @param cfg the configuration to be looked up
+ *
+ * @return array of values, null if the key was not found
+ */
+char** get_configuration_values(const char *key, const char* section,
+ const struct configuration *cfg);
-char ** extract_values_delim(char *value, const char *delim);
+/**
+ * Function to get the values for a given key in the specified section in the
+ * specified configuration. The value is split by the specified delimiter.
+ * It's up to the caller to free the memory used for storing the return values.
+ *
+ * @param key the key to be looked up
+ * @param section the name of the section to be looked up
+ * @param cfg the section to be looked up
+ * @param delimiter the delimiter to be used to split the value
+ *
+ * @return array of values, null if the key was not found
+ */
+char** get_configuration_values_delimiter(const char *key, const char* section,
+ const struct configuration *cfg, const char *delimiter);
-// free the memory returned by get_values
-void free_values(char** values);
+/**
+ * Function to retrieve the specified section from the configuration.
+ *
+ * @param section the name of the section to retrieve
+ * @param cfg the configuration structure to use
+ *
+ * @return pointer to section struct containing details of the section
+ * null on error
+ */
+struct section* get_configuration_section(const char *section,
+ const struct configuration *cfg);
-//method to free allocated configuration
-void free_configurations(struct configuration *cfg);
+/**
+ * Method to free an allocated config struct.
+ *
+ * @param cfg pointer to the structure to free
+ */
+void free_configuration(struct configuration *cfg);
/**
* If str is a string of the form key=val, find 'key'
@@ -106,11 +207,4 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
-/**
- * Trim whitespace from beginning and end.
- *
- * @param input Input string that needs to be trimmed
- *
- * @return the trimmed string allocated with malloc. I has to be freed by the caller
-*/
-char* trim(char* input);
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index def628e..9f754c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -19,6 +19,8 @@
#include "configuration.h"
#include "container-executor.h"
#include "utils/string-utils.h"
+#include "util.h"
+#include "config.h"
#include <inttypes.h>
#include <libgen.h>
@@ -43,8 +45,6 @@
#include <getopt.h>
#include <regex.h>
-#include "config.h"
-
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
#endif
@@ -92,7 +92,8 @@ FILE* ERRORFILE = NULL;
static uid_t nm_uid = -1;
static gid_t nm_gid = -1;
-struct configuration executor_cfg = {.size=0, .confdetails=NULL};
+struct configuration CFG = {.size=0, .sections=NULL};
+struct section executor_cfg = {.size=0, .kv_pairs=NULL};
char *concatenate(char *concat_pattern, char *return_path_name,
int numArgs, ...);
@@ -103,18 +104,25 @@ void set_nm_uid(uid_t user, gid_t group) {
}
//function used to load the configurations present in the secure config
-void read_executor_config(const char* file_name) {
- read_config(file_name, &executor_cfg);
+void read_executor_config(const char *file_name) {
+ const struct section *tmp = NULL;
+ int ret = read_config(file_name, &CFG);
+ if (ret == 0) {
+ tmp = get_configuration_section("", &CFG);
+ if (tmp != NULL) {
+ executor_cfg = *tmp;
+ }
+ }
}
//function used to free executor configuration data
void free_executor_configurations() {
- free_configurations(&executor_cfg);
+ free_configuration(&CFG);
}
//Lookup nodemanager group from container executor configuration.
char *get_nodemanager_group() {
- return get_value(NM_GROUP_KEY, &executor_cfg);
+ return get_section_value(NM_GROUP_KEY, &executor_cfg);
}
int check_executor_permissions(char *executable_file) {
@@ -431,8 +439,8 @@ int change_user(uid_t user, gid_t group) {
}
int is_feature_enabled(const char* feature_key, int default_value,
- struct configuration *cfg) {
- char *enabled_str = get_value(feature_key, cfg);
+ struct section *cfg) {
+ char *enabled_str = get_section_value(feature_key, cfg);
int enabled = default_value;
if (enabled_str != NULL) {
@@ -753,7 +761,7 @@ static struct passwd* get_user_info(const char* user) {
}
int is_whitelisted(const char *user) {
- char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg);
+ char **whitelist = get_section_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg);
char **users = whitelist;
if (whitelist != NULL) {
for(; *users; ++users) {
@@ -781,7 +789,7 @@ struct passwd* check_user(const char *user) {
fflush(LOGFILE);
return NULL;
}
- char *min_uid_str = get_value(MIN_USERID_KEY, &executor_cfg);
+ char *min_uid_str = get_section_value(MIN_USERID_KEY, &executor_cfg);
int min_uid = DEFAULT_MIN_USERID;
if (min_uid_str != NULL) {
char *end_ptr = NULL;
@@ -808,7 +816,7 @@ struct passwd* check_user(const char *user) {
free(user_info);
return NULL;
}
- char **banned_users = get_values(BANNED_USERS_KEY, &executor_cfg);
+ char **banned_users = get_section_values(BANNED_USERS_KEY, &executor_cfg);
banned_users = banned_users == NULL ?
(char**) DEFAULT_BANNED_USERS : banned_users;
char **banned_user = banned_users;
@@ -1194,7 +1202,6 @@ char** tokenize_docker_command(const char *input, int *split_counter) {
char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
char **linesplit = (char **) malloc(sizeof(char *));
char *p = NULL;
- int c = 0;
*split_counter = 0;
strncpy(line, input, strlen(input));
@@ -1408,12 +1415,12 @@ char* parse_docker_command_file(const char* command_file) {
int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
+ char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
docker_binary = check_docker_binary(docker_binary);
char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX);
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
- char **args = extract_values_delim(docker_command_with_binary, " ");
+ char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
@@ -1574,7 +1581,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
uid_t prev_uid = geteuid();
char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
+ char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
docker_binary = check_docker_binary(docker_binary);
fprintf(LOGFILE, "Creating script paths...\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 1dc0491..ea8b5e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -35,51 +35,6 @@ enum command {
LIST_AS_USER = 5
};
-enum errorcodes {
- INVALID_ARGUMENT_NUMBER = 1,
- //INVALID_USER_NAME 2
- INVALID_COMMAND_PROVIDED = 3,
- // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4
- INVALID_NM_ROOT_DIRS = 5,
- SETUID_OPER_FAILED, //6
- UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7
- UNABLE_TO_SIGNAL_CONTAINER, //8
- INVALID_CONTAINER_PID, //9
- // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10
- // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11
- // UNABLE_TO_STAT_FILE (NOT USED) 12
- // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13
- // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14
- // INITIALIZE_CONTAINER_FAILED (NOT USED) 15
- // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16
- // INVALID_LOG_DIR (NOT USED) 17
- OUT_OF_MEMORY = 18,
- // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19
- INITIALIZE_USER_FAILED = 20,
- PATH_TO_DELETE_IS_NULL, //21
- INVALID_CONTAINER_EXEC_PERMISSIONS, //22
- // PREPARE_JOB_LOGS_FAILED (NOT USED) 23
- INVALID_CONFIG_FILE = 24,
- SETSID_OPER_FAILED = 25,
- WRITE_PIDFILE_FAILED = 26,
- WRITE_CGROUP_FAILED = 27,
- TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
- DOCKER_RUN_FAILED = 29,
- ERROR_OPENING_DOCKER_FILE = 30,
- ERROR_READING_DOCKER_FILE = 31,
- FEATURE_DISABLED = 32,
- COULD_NOT_CREATE_SCRIPT_COPY = 33,
- COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
- COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
- COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
- COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
- ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
- ERROR_SANITIZING_DOCKER_COMMAND = 39,
- DOCKER_IMAGE_INVALID = 40,
- DOCKER_CONTAINER_NAME_INVALID = 41,
- ERROR_COMPILING_REGEX = 42
-};
-
enum operations {
CHECK_SETUP = 1,
MOUNT_CGROUPS = 2,
@@ -111,11 +66,6 @@ enum operations {
extern struct passwd *user_detail;
-// the log file for messages
-extern FILE *LOGFILE;
-// the log file for error messages
-extern FILE *ERRORFILE;
-
// get the executable's filename
char* get_executable(char *argv0);
@@ -276,7 +226,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
/** Check if a feature is enabled in the specified configuration. */
int is_feature_enabled(const char* feature_key, int default_value,
- struct configuration *cfg);
+ struct section *cfg);
/** Check if tc (traffic control) support is enabled in configuration. */
int is_tc_support_enabled();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index ce46b77..55973a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -31,6 +31,7 @@
#include "config.h"
#include "configuration.h"
#include "container-executor.h"
+#include "util.h"
#include <errno.h>
#include <stdio.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index fdc0496..b2187c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -19,6 +19,7 @@
#include "config.h"
#include "configuration.h"
#include "container-executor.h"
+#include "util.h"
#include <errno.h>
#include <grp.h>
@@ -420,7 +421,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.resources_key = resources_key;
cmd_input.resources_value = resources_value;
- cmd_input.resources_values = extract_values(resources_value);
+ cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
return 0;
} else {
@@ -471,7 +472,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.resources_key = resources_key;
cmd_input.resources_value = resources_value;
- cmd_input.resources_values = extract_values(resources_value);
+ cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_CONTAINER;
return 0;
@@ -565,8 +566,8 @@ int main(int argc, char **argv) {
exit_code = initialize_app(cmd_input.yarn_user_name,
cmd_input.app_id,
cmd_input.cred_file,
- extract_values(cmd_input.local_dirs),
- extract_values(cmd_input.log_dirs),
+ split(cmd_input.local_dirs),
+ split(cmd_input.log_dirs),
argv + optind);
break;
case RUN_AS_USER_LAUNCH_DOCKER_CONTAINER:
@@ -591,8 +592,8 @@ int main(int argc, char **argv) {
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
- extract_values(cmd_input.local_dirs),
- extract_values(cmd_input.log_dirs),
+ split(cmd_input.local_dirs),
+ split(cmd_input.log_dirs),
cmd_input.docker_command_file,
cmd_input.resources_key,
cmd_input.resources_values);
@@ -619,8 +620,8 @@ int main(int argc, char **argv) {
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
- extract_values(cmd_input.local_dirs),
- extract_values(cmd_input.log_dirs),
+ split(cmd_input.local_dirs),
+ split(cmd_input.log_dirs),
cmd_input.resources_key,
cmd_input.resources_values);
free(cmd_input.resources_key);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
new file mode 100644
index 0000000..8e39ca8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "util.h"
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+char** split_delimiter(char *value, const char *delim) {
+ char **return_values = NULL;
+ char *temp_tok = NULL;
+ char *tempstr = NULL;
+ int size = 0;
+ int per_alloc_size = 10;
+ int return_values_size = per_alloc_size;
+ int failed = 0;
+
+ //first allocate any array of 10
+ if(value != NULL) {
+ return_values = (char **) malloc(sizeof(char *) * return_values_size);
+ if (!return_values) {
+ fprintf(ERRORFILE, "Allocation error for return_values in %s.\n",
+ __func__);
+ failed = 1;
+ goto cleanup;
+ }
+ memset(return_values, 0, sizeof(char *) * return_values_size);
+
+ temp_tok = strtok_r(value, delim, &tempstr);
+ while (temp_tok != NULL) {
+ temp_tok = strdup(temp_tok);
+ if (NULL == temp_tok) {
+ fprintf(ERRORFILE, "Allocation error in %s.\n", __func__);
+ failed = 1;
+ goto cleanup;
+ }
+
+ return_values[size++] = temp_tok;
+
+ // Make sure returned values has enough space for the trailing NULL.
+ if (size >= return_values_size - 1) {
+ return_values_size += per_alloc_size;
+ return_values = (char **) realloc(return_values,(sizeof(char *) *
+ return_values_size));
+
+ // Make sure new added memory are filled with NULL
+ for (int i = size; i < return_values_size; i++) {
+ return_values[i] = NULL;
+ }
+ }
+ temp_tok = strtok_r(NULL, delim, &tempstr);
+ }
+ }
+
+ // Put trailing NULL to indicate values terminates.
+ if (return_values != NULL) {
+ return_values[size] = NULL;
+ }
+
+cleanup:
+ if (failed) {
+ free_values(return_values);
+ return NULL;
+ }
+
+ return return_values;
+}
+
+/**
+ * Extracts array of values from the '%' separated list of values.
+ */
+char** split(char *value) {
+ return split_delimiter(value, "%");
+}
+
+// free an entry set of values
+void free_values(char** values) {
+ if (values != NULL) {
+ int idx = 0;
+ while (values[idx]) {
+ free(values[idx]);
+ idx++;
+ }
+ free(values);
+ }
+}
+
+/**
+ * Trim whitespace from beginning and end.
+*/
+char* trim(const char* input) {
+ const char *val_begin;
+ const char *val_end;
+ char *ret;
+
+ if (input == NULL) {
+ return NULL;
+ }
+
+ val_begin = input;
+ val_end = input + strlen(input);
+
+ while (val_begin < val_end && isspace(*val_begin))
+ val_begin++;
+ while (val_end > val_begin && isspace(*(val_end - 1)))
+ val_end--;
+
+ ret = (char *) malloc(
+ sizeof(char) * (val_end - val_begin + 1));
+ if (ret == NULL) {
+ fprintf(ERRORFILE, "Allocation error\n");
+ exit(OUT_OF_MEMORY);
+ }
+
+ strncpy(ret, val_begin, val_end - val_begin);
+ ret[val_end - val_begin] = '\0';
+ return ret;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
new file mode 100644
index 0000000..a8a12a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+
+#include <stdio.h>
+
+enum errorcodes {
+ INVALID_ARGUMENT_NUMBER = 1,
+ //INVALID_USER_NAME 2
+ INVALID_COMMAND_PROVIDED = 3,
+ // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4
+ INVALID_NM_ROOT_DIRS = 5,
+ SETUID_OPER_FAILED, //6
+ UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7
+ UNABLE_TO_SIGNAL_CONTAINER, //8
+ INVALID_CONTAINER_PID, //9
+ // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10
+ // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11
+ // UNABLE_TO_STAT_FILE (NOT USED) 12
+ // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13
+ // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14
+ // INITIALIZE_CONTAINER_FAILED (NOT USED) 15
+ // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16
+ // INVALID_LOG_DIR (NOT USED) 17
+ OUT_OF_MEMORY = 18,
+ // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19
+ INITIALIZE_USER_FAILED = 20,
+ PATH_TO_DELETE_IS_NULL, //21
+ INVALID_CONTAINER_EXEC_PERMISSIONS, //22
+ // PREPARE_JOB_LOGS_FAILED (NOT USED) 23
+ INVALID_CONFIG_FILE = 24,
+ SETSID_OPER_FAILED = 25,
+ WRITE_PIDFILE_FAILED = 26,
+ WRITE_CGROUP_FAILED = 27,
+ TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
+ DOCKER_RUN_FAILED = 29,
+ ERROR_OPENING_DOCKER_FILE = 30,
+ ERROR_READING_DOCKER_FILE = 31,
+ FEATURE_DISABLED = 32,
+ COULD_NOT_CREATE_SCRIPT_COPY = 33,
+ COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
+ COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
+ COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
+ COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
+ ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
+ ERROR_SANITIZING_DOCKER_COMMAND = 39,
+ DOCKER_IMAGE_INVALID = 40,
+ DOCKER_CONTAINER_NAME_INVALID = 41,
+ ERROR_COMPILING_REGEX = 42
+};
+
+
+// the log file for messages
+extern FILE *LOGFILE;
+// the log file for error messages
+extern FILE *ERRORFILE;
+/**
+ * Function to split the given string using '%' as the separator. It's
+ * up to the caller to free the memory for the returned array. Use the
+ * free_values function to free the allocated memory.
+ *
+ * @param str the string to split
+ *
+ * @return an array of strings
+ */
+char** split(char *str);
+
+/**
+ * Function to split the given string using the delimiter specified. It's
+ * up to the caller to free the memory for the returned array. Use the
+ * free_values function to free the allocated memory.
+ *
+ * @param str the string to split
+ * @param delimiter the delimiter to use
+ *
+ * @return an array of strings
+ */
+char** split_delimiter(char *value, const char *delimiter);
+
+/**
+ * Function to free an array of strings.
+ *
+ * @param values the array to free
+ *
+ */
+void free_values(char **values);
+
+/**
+ * Trim whitespace from beginning and end. The returned string has to be freed
+ * by the caller.
+ *
+ * @param input Input string that needs to be trimmed
+ *
+ * @return the trimmed string allocated with malloc
+*/
+char* trim(const char *input);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg
new file mode 100644
index 0000000..4d0b90d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[section-1]
+key1=value1
+split-key=val1,val2,val3
+perc-key=perc-val1%perc-val2
+# some comment
+
+[split-section]
+key3=value3
+
+[section-2]
+key1=value2
+
+key2=value2
+
+[split-section]
+key4=value4
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg
new file mode 100644
index 0000000..aa02db8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test mixed mode config file
+# Initial few lines are in the key=value format
+# and then the sections start
+
+key1=value1
+key2=value2
+
+
+[section-1]
+key3=value3
+key1=value4
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg
new file mode 100644
index 0000000..947a3fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+yarn.nodemanager.linux-container-executor.group=yarn
+banned.users=root,testuser1,testuser2#comma separated list of users who can not run applications
+min.user.id=1000
+allowed.system.users=nobody,daemon
+feature.docker.enabled=1
+feature.tc.enabled=0
+docker.binary=/usr/bin/docker
+yarn.local.dirs=/var/run/yarn%/tmp/mydir
+test.key=#no value for this key
+# test.key2=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec694145/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 3202652..3cfefa0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -18,6 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
#include "utils/string-utils.h"
+#include "util.h"
#include <inttypes.h>
#include <errno.h>
@@ -404,7 +405,7 @@ void test_delete_app() {
}
void validate_feature_enabled_value(int expected_value, const char* key,
- int default_value, struct configuration *cfg) {
+ int default_value, struct section *cfg) {
int value = is_feature_enabled(key, default_value, cfg);
if (value != expected_value) {
@@ -419,7 +420,8 @@ void test_is_feature_enabled() {
FILE *file = fopen(filename, "w");
int disabled = 0;
int enabled = 1;
- struct configuration cfg = {.size=0, .confdetails=NULL};
+ struct configuration exec_cfg = {.size=0, .sections=NULL};
+ struct section cfg = {.size=0, .kv_pairs=NULL};
if (file == NULL) {
printf("FAIL: Could not open configuration file: %s\n", filename);
@@ -433,7 +435,8 @@ void test_is_feature_enabled() {
fprintf(file, "feature.name5.enabled=-1\n");
fprintf(file, "feature.name6.enabled=2\n");
fclose(file);
- read_config(filename, &cfg);
+ read_config(filename, &exec_cfg);
+ cfg = *(get_configuration_section("", &exec_cfg));
validate_feature_enabled_value(disabled, "feature.name1.enabled",
disabled, &cfg);
@@ -449,7 +452,7 @@ void test_is_feature_enabled() {
disabled, &cfg);
- free_configurations(&cfg);
+ free_configuration(&exec_cfg);
}
void test_delete_user() {
@@ -1345,8 +1348,8 @@ int main(int argc, char **argv) {
read_executor_config(TEST_ROOT "/test.cfg");
- local_dirs = extract_values(strdup(NM_LOCAL_DIRS));
- log_dirs = extract_values(strdup(NM_LOG_DIRS));
+ local_dirs = split(strdup(NM_LOCAL_DIRS));
+ log_dirs = split(strdup(NM_LOG_DIRS));
create_nm_roots(local_dirs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org