You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/06/07 21:15:00 UTC
[2/4] hadoop git commit: Sanitize arguments before launching Docker
containers.
Sanitize arguments before launching Docker containers.
(cherry picked from commit 23bbbce63e963284219a68f992f5febc73300d7d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12ce6af1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12ce6af1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12ce6af1
Branch: refs/heads/branch-2.8.1-private
Commit: 12ce6af15d36d9230fcdd1e651e645d0972f3eca
Parents: bbe3b08
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu May 18 12:03:52 2017 +0530
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Wed Jun 7 13:37:41 2017 -0700
----------------------------------------------------------------------
.../impl/container-executor.c | 173 ++++++++++++++++++-
.../impl/container-executor.h | 8 +-
.../test/test-container-executor.c | 44 +++++
3 files changed, 223 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ce6af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index bdbcdfa..7aa36cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -40,6 +40,7 @@
#include <sys/stat.h>
#include <sys/mount.h>
#include <sys/wait.h>
+#include <getopt.h>
#include "config.h"
@@ -1117,7 +1118,170 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
+static char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) calloc(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ ret[i + 1] = '\0';
+ i++;
+ }
+ p++;
+ }
+ return ret;
+}
+
+static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ strcat(*str, param);
+ strcat(*str, "'");
+ if(strlen(*str) + strlen(tmp) > *size) {
+ *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
+ if(*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ *size = strlen(*str) + strlen(tmp) + 1024;
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
+
+char** tokenize_docker_command(const char *input, int *split_counter) {
+ char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
+ char **linesplit = (char **) malloc(sizeof(char *));
+ char *p = NULL;
+ int c = 0;
+ *split_counter = 0;
+ strncpy(line, input, strlen(input));
+
+ p = strtok(line, " ");
+ while(p != NULL) {
+ linesplit[*split_counter] = p;
+ (*split_counter)++;
+ linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
+ if(linesplit == NULL) {
+ fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
+ strerror(errno));
+ fflush(ERRORFILE);
+ exit(OUT_OF_MEMORY);
+ }
+ p = strtok(NULL, " ");
+ }
+ linesplit[*split_counter] = NULL;
+ return linesplit;
+}
+
+char* sanitize_docker_command(const char *line) {
+ static struct option long_options[] = {
+ {"name", required_argument, 0, 'n' },
+ {"user", required_argument, 0, 'u' },
+ {"rm", no_argument, 0, 'r' },
+ {"workdir", required_argument, 0, 'w' },
+ {"net", required_argument, 0, 'e' },
+ {"cgroup-parent", required_argument, 0, 'g' },
+ {"privileged", no_argument, 0, 'p' },
+ {"cap-add", required_argument, 0, 'a' },
+ {"cap-drop", required_argument, 0, 'o' },
+ {"device", required_argument, 0, 'i' },
+ {"detach", required_argument, 0, 't' },
+ {0, 0, 0, 0}
+ };
+
+ int c = 0;
+ int option_index = 0;
+ char *output = NULL;
+ size_t output_size = 0;
+ char **linesplit;
+ int split_counter = 0;
+ int len = strlen(line);
+
+ linesplit = tokenize_docker_command(line, &split_counter);
+
+ output_size = len * 2;
+ output = (char *) calloc(output_size, sizeof(char));
+ if(output == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ strcat(output, linesplit[0]);
+ strcat(output, " ");
+ optind = 1;
+ while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
+ switch(c) {
+ case 'n':
+ quote_and_append_arg(&output, &output_size, "--name=", optarg);
+ break;
+ case 'w':
+ quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
+ break;
+ case 'u':
+ quote_and_append_arg(&output, &output_size, "--user=", optarg);
+ break;
+ case 'e':
+ quote_and_append_arg(&output, &output_size, "--net=", optarg);
+ break;
+ case 'v':
+ quote_and_append_arg(&output, &output_size, "-v ", optarg);
+ break;
+ case 'a':
+ quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
+ break;
+ case 'o':
+ quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
+ break;
+ case 'd':
+ strcat(output, "-d ");
+ break;
+ case 'r':
+ strcat(output, "--rm ");
+ break;
+ case 'g':
+ quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
+ break;
+ case 'p':
+ strcat(output, "--privileged ");
+ break;
+ case 'i':
+ quote_and_append_arg(&output, &output_size, "--device=", optarg);
+ break;
+ case 't':
+ quote_and_append_arg(&output, &output_size, "--detach=", optarg);
+ break;
+ default:
+ fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
+ fflush(LOGFILE);
+ return NULL;
+ break;
+ }
+ }
+
+ if(optind < split_counter) {
+ quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
+ strcat(output, "'");
+ while(optind < split_counter) {
+ strcat(output, linesplit[optind++]);
+ strcat(output, " ");
+ }
+ strcat(output, "'");
+ }
+
+ return output;
+}
+
char* parse_docker_command_file(const char* command_file) {
+
size_t len = 0;
char *line = NULL;
ssize_t read;
@@ -1136,7 +1300,14 @@ char* parse_docker_command_file(const char* command_file) {
}
fclose(stream);
- return line;
+ char* ret = sanitize_docker_command(line);
+ if(ret == NULL) {
+ exit(ERROR_SANITIZING_DOCKER_COMMAND);
+ }
+ fprintf(LOGFILE, "Using command %s\n", ret);
+ fflush(LOGFILE);
+
+ return ret;
}
int run_docker(const char *command_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ce6af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 1c64c22..4d0d75b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -66,7 +66,8 @@ enum errorcodes {
DOCKER_RUN_FAILED=29,
ERROR_OPENING_FILE = 30,
ERROR_READING_FILE = 31,
- FEATURE_DISABLED = 32
+ FEATURE_DISABLED = 32,
+ ERROR_SANITIZING_DOCKER_COMMAND = 33
};
enum operations {
@@ -288,3 +289,8 @@ int traffic_control_read_stats(char *command_file);
* Run a docker command passing the command file as an argument
*/
int run_docker(const char *command_file);
+
+/**
+ * Sanitize docker commands. Returns NULL if there was any failure.
+*/
+char* sanitize_docker_command(const char *line);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ce6af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 22941b1..fcc05a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -713,6 +713,47 @@ void test_run_container() {
check_pid_file(cgroups_pids[1], child);
}
+void test_sanitize_docker_command() {
+
+/*
+ char *input[] = {
+ "run ''''''''"
+ };
+*/
+ char *input[] = {
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
+ "run ''''''''"
+ };
+/*
+ char *expected_output[] = {
+ "run ''''''''"
+ };
+*/
+ char *expected_output[] = {
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '|| touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh '",
+ "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ''",
+ };
+
+ int input_size = sizeof(input) / sizeof(char *);
+ int i = 0;
+ for(i = 0; i < input_size; i++) {
+ char *command = (char *) calloc(strlen(input[i]), sizeof(char));
+ strncpy(command, input[i], strlen(input[i]));
+ char *op = sanitize_docker_command(command);
+ if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
+ printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
+ exit(1);
+ }
+ free(command);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -798,6 +839,9 @@ int main(int argc, char **argv) {
printf("\nTesting delete_app()\n");
test_delete_app();
+ printf("\nTesting sanitize docker commands()\n");
+ test_sanitize_docker_command();
+
test_check_user(0);
#ifdef __APPLE__
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org