You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2017/05/18 06:54:01 UTC
[4/8] hadoop git commit: Sanitize arguments before launching Docker
containers.
Sanitize arguments before launching Docker containers.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd1a04e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd1a04e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd1a04e3
Branch: refs/heads/branch-2
Commit: cd1a04e3c6bc4432bb16adb1bd5bf317e1b0405e
Parents: 603a0a3
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu May 18 11:46:03 2017 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu May 18 11:46:03 2017 +0530
----------------------------------------------------------------------
.../impl/container-executor.c | 173 ++++++++++++++++++-
.../impl/container-executor.h | 6 +
.../test/test-container-executor.c | 44 +++++
3 files changed, 222 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd1a04e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 587adf7..618c602 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -39,6 +39,7 @@
#include <sys/stat.h>
#include <sys/mount.h>
#include <sys/wait.h>
+#include <getopt.h>
#include "config.h"
@@ -1137,7 +1138,170 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
+static char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) calloc(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ ret[i + 1] = '\0';
+ i++;
+ }
+ p++;
+ }
+ return ret;
+}
+
+static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ strcat(*str, param);
+ strcat(*str, "'");
+ if(strlen(*str) + strlen(tmp) > *size) {
+ *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
+ if(*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ *size = strlen(*str) + strlen(tmp) + 1024;
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
+
+char** tokenize_docker_command(const char *input, int *split_counter) {
+ char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
+ char **linesplit = (char **) malloc(sizeof(char *));
+ char *p = NULL;
+ int c = 0;
+ *split_counter = 0;
+ strncpy(line, input, strlen(input));
+
+ p = strtok(line, " ");
+ while(p != NULL) {
+ linesplit[*split_counter] = p;
+ (*split_counter)++;
+ linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
+ if(linesplit == NULL) {
+ fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
+ strerror(errno));
+ fflush(ERRORFILE);
+ exit(OUT_OF_MEMORY);
+ }
+ p = strtok(NULL, " ");
+ }
+ linesplit[*split_counter] = NULL;
+ return linesplit;
+}
+
+char* sanitize_docker_command(const char *line) {
+ static struct option long_options[] = {
+ {"name", required_argument, 0, 'n' },
+ {"user", required_argument, 0, 'u' },
+ {"rm", no_argument, 0, 'r' },
+ {"workdir", required_argument, 0, 'w' },
+ {"net", required_argument, 0, 'e' },
+ {"cgroup-parent", required_argument, 0, 'g' },
+ {"privileged", no_argument, 0, 'p' },
+ {"cap-add", required_argument, 0, 'a' },
+ {"cap-drop", required_argument, 0, 'o' },
+ {"device", required_argument, 0, 'i' },
+ {"detach", required_argument, 0, 't' },
+ {0, 0, 0, 0}
+ };
+
+ int c = 0;
+ int option_index = 0;
+ char *output = NULL;
+ size_t output_size = 0;
+ char **linesplit;
+ int split_counter = 0;
+ int len = strlen(line);
+
+ linesplit = tokenize_docker_command(line, &split_counter);
+
+ output_size = len * 2;
+ output = (char *) calloc(output_size, sizeof(char));
+ if(output == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ strcat(output, linesplit[0]);
+ strcat(output, " ");
+ optind = 1;
+ while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
+ switch(c) {
+ case 'n':
+ quote_and_append_arg(&output, &output_size, "--name=", optarg);
+ break;
+ case 'w':
+ quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
+ break;
+ case 'u':
+ quote_and_append_arg(&output, &output_size, "--user=", optarg);
+ break;
+ case 'e':
+ quote_and_append_arg(&output, &output_size, "--net=", optarg);
+ break;
+ case 'v':
+ quote_and_append_arg(&output, &output_size, "-v ", optarg);
+ break;
+ case 'a':
+ quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
+ break;
+ case 'o':
+ quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
+ break;
+ case 'd':
+ strcat(output, "-d ");
+ break;
+ case 'r':
+ strcat(output, "--rm ");
+ break;
+ case 'g':
+ quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
+ break;
+ case 'p':
+ strcat(output, "--privileged ");
+ break;
+ case 'i':
+ quote_and_append_arg(&output, &output_size, "--device=", optarg);
+ break;
+ case 't':
+ quote_and_append_arg(&output, &output_size, "--detach=", optarg);
+ break;
+ default:
+ fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
+ fflush(LOGFILE);
+ return NULL;
+ break;
+ }
+ }
+
+ if(optind < split_counter) {
+ quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
+ strcat(output, "'");
+ while(optind < split_counter) {
+ strcat(output, linesplit[optind++]);
+ strcat(output, " ");
+ }
+ strcat(output, "'");
+ }
+
+ return output;
+}
+
char* parse_docker_command_file(const char* command_file) {
+
size_t len = 0;
char *line = NULL;
ssize_t read;
@@ -1156,7 +1320,14 @@ char* parse_docker_command_file(const char* command_file) {
}
fclose(stream);
- return line;
+ char* ret = sanitize_docker_command(line);
+ if(ret == NULL) {
+ exit(ERROR_SANITIZING_DOCKER_COMMAND);
+ }
+ fprintf(LOGFILE, "Using command %s\n", ret);
+ fflush(LOGFILE);
+
+ return ret;
}
int run_docker(const char *command_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd1a04e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 826ff2c..e40bd90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -74,6 +74,7 @@ enum errorcodes {
COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
+ ERROR_SANITIZING_DOCKER_COMMAND = 39
};
enum operations {
@@ -303,3 +304,8 @@ int is_docker_support_enabled();
* Run a docker command passing the command file as an argument
*/
int run_docker(const char *command_file);
+
+/**
+ * Sanitize docker commands. Returns NULL if there was any failure.
+*/
+char* sanitize_docker_command(const char *line);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd1a04e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 4c22a5f..fd99325 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -1013,6 +1013,47 @@ void test_recursive_unlink_children() {
}
}
+void test_sanitize_docker_command() {
+
+/*
+ char *input[] = {
+ "run ''''''''"
+ };
+*/
+ char *input[] = {
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run ''''''''"
+ };
+/*
+ char *expected_output[] = {
+ "run ''''''''"
+ };
+*/
+ char *expected_output[] = {
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ''",
+ };
+
+ int input_size = sizeof(input) / sizeof(char *);
+ int i = 0;
+ for(i = 0; i < input_size; i++) {
+ char *command = (char *) calloc(strlen(input[i]), sizeof(char));
+ strncpy(command, input[i], strlen(input[i]));
+ char *op = sanitize_docker_command(command);
+ if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
+ printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
+ exit(1);
+ }
+ free(command);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1104,6 +1145,9 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
+ printf("\nTesting sanitize docker commands()\n");
+ test_sanitize_docker_command();
+
test_check_user(0);
#ifdef __APPLE__
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org