You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2018/08/16 15:09:43 UTC
hadoop git commit: YARN-8656. container-executor should not write
cgroup tasks files for docker containers. Contributed by Jim Brennan
Repository: hadoop
Updated Branches:
refs/heads/trunk 6df606f1b -> cb21eaa02
YARN-8656. container-executor should not write cgroup tasks files for docker containers. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb21eaa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb21eaa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb21eaa0
Branch: refs/heads/trunk
Commit: cb21eaa026d80a2c9836030d959c0dd7f87c4d6b
Parents: 6df606f
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Aug 16 10:06:17 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Aug 16 10:06:17 2018 -0500
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 4 +--
.../impl/container-executor.c | 21 +---------------
.../impl/container-executor.h | 3 +--
.../main/native/container-executor/impl/main.c | 26 +++-----------------
.../runtime/TestDockerContainerRuntime.java | 8 +++---
5 files changed, 10 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb21eaa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 5d6f61e..1872830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1156,7 +1156,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
- String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
PrivilegedOperation launchOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
@@ -1174,8 +1173,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs),
- commandFile,
- resourcesOpts);
+ commandFile);
String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb21eaa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 6734b94..f8b89ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1547,9 +1547,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
const char *pid_file, char* const* local_dirs,
- char* const* log_dirs, const char *command_file,
- const char *resources_key,
- char* const* resources_values) {
+ char* const* log_dirs, const char *command_file) {
int exit_code = -1;
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
@@ -1732,23 +1730,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
if (pid != 0) {
-#ifdef __linux
- fprintf(LOGFILE, "Writing to cgroup task files...\n");
- // cgroups-based resource enforcement
- if (resources_key != NULL && ! strcmp(resources_key, "cgroups")) {
- // write pid to cgroups
- char* const* cgroup_ptr;
- for (cgroup_ptr = resources_values; cgroup_ptr != NULL &&
- *cgroup_ptr != NULL; ++cgroup_ptr) {
- if (strcmp(*cgroup_ptr, "none") != 0 &&
- write_pid_to_cgroup_as_root(*cgroup_ptr, pid) != 0) {
- exit_code = WRITE_CGROUP_FAILED;
- goto cleanup;
- }
- }
- }
-#endif
-
// write pid to pidfile
fprintf(LOGFILE, "Writing pid file...\n");
if (pid_file == NULL
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb21eaa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 32e953d..002f85f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -104,8 +104,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
const char *script_name, const char *cred_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs,
- const char *command_file,const char *resources_key,
- char* const* resources_values);
+ const char *command_file);
/*
* Function used to launch a container as the provided user. It does the following :
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb21eaa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 76fa39f..93691f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -430,8 +430,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
case LAUNCH_DOCKER_CONTAINER:
if(is_docker_support_enabled()) {
//kill me now.
- if (!(argc == 14 || argc == 15)) {
- fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for"
+ if (!(argc == 13 || argc == 14)) {
+ fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for"
" launch docker container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
@@ -448,21 +448,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
// good log dirs as a comma separated list
cmd_input.log_dirs = argv[optind++];
cmd_input.docker_command_file = argv[optind++];
- // key,value pair describing resources
- resources = argv[optind++];
- resources_key = malloc(strlen(resources));
- resources_value = malloc(strlen(resources));
- if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
- get_kv_value(resources, resources_value, strlen(resources)) < 0) {
- fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
- resources);
- fflush(ERRORFILE);
- free(resources_key);
- free(resources_value);
- return INVALID_ARGUMENT_NUMBER;
- }
//network isolation through tc
- if (argc == 15) {
+ if (argc == 14) {
if(is_tc_support_enabled()) {
cmd_input.traffic_control_command_file = argv[optind++];
} else {
@@ -471,9 +458,6 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
}
}
- cmd_input.resources_key = resources_key;
- cmd_input.resources_value = resources_value;
- cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
return 0;
} else {
@@ -653,9 +637,7 @@ int main(int argc, char **argv) {
cmd_input.pid_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
- cmd_input.docker_command_file,
- cmd_input.resources_key,
- cmd_input.resources_values);
+ cmd_input.docker_command_file);
break;
case RUN_AS_USER_LAUNCH_CONTAINER:
if (cmd_input.traffic_control_command_file != NULL) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb21eaa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 80823d3..e098778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -355,9 +355,9 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
- //This invocation of container-executor should use 13 arguments in a
+ //This invocation of container-executor should use 12 arguments in a
// specific order
- int expected = 13;
+ int expected = 12;
int counter = 1;
Assert.assertEquals(expected, args.size());
Assert.assertEquals(user, args.get(counter++));
@@ -373,7 +373,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(pidFilePath.toString(), args.get(counter++));
Assert.assertEquals(localDirs.get(0), args.get(counter++));
Assert.assertEquals(logDirs.get(0), args.get(counter++));
- Assert.assertEquals(resourcesOptions, args.get(++counter));
return op;
}
@@ -2116,7 +2115,7 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
- int expectedArgs = 13;
+ int expectedArgs = 12;
int argsCounter = 0;
Assert.assertEquals(expectedArgs, args.size());
Assert.assertEquals(runAsUser, args.get(argsCounter++));
@@ -2134,7 +2133,6 @@ public class TestDockerContainerRuntime {
Assert.assertEquals(localDirs.get(0), args.get(argsCounter++));
Assert.assertEquals(logDirs.get(0), args.get(argsCounter++));
String dockerCommandFile = args.get(argsCounter++);
- Assert.assertEquals(resourcesOptions, args.get(argsCounter));
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org