You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/05/04 19:28:01 UTC
[33/50] [abbrv] hadoop git commit: YARN-8209. Fixed NPE in Yarn
Service deletion. Contributed by Eric Badger
YARN-8209. Fixed NPE in Yarn Service deletion.
Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/883f6822
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/883f6822
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/883f6822
Branch: refs/heads/HDFS-12943
Commit: 883f68222a9cfd06f79a8fcd75ec9fef00abc035
Parents: 19ae588
Author: Eric Yang <ey...@apache.org>
Authored: Wed May 2 14:33:31 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed May 2 14:33:31 2018 -0400
----------------------------------------------------------------------
.../linux/privileged/PrivilegedOperation.java | 4 +-
.../runtime/DockerLinuxContainerRuntime.java | 21 +++++----
.../linux/runtime/docker/DockerClient.java | 7 ++-
.../linux/runtime/docker/DockerCommand.java | 32 +++++++++++++
.../runtime/docker/DockerCommandExecutor.java | 12 ++---
.../runtime/docker/DockerInspectCommand.java | 19 ++++++++
.../linux/runtime/docker/DockerRmCommand.java | 16 +++++++
.../impl/container-executor.c | 28 ++++++++++++
.../impl/container-executor.h | 10 ++++-
.../main/native/container-executor/impl/main.c | 47 ++++++++++++++++++--
.../docker/TestDockerCommandExecutor.java | 31 +++++++------
11 files changed, 184 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
index 189c0d0..92a82e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
@@ -54,7 +54,9 @@ public class PrivilegedOperation {
GPU("--module-gpu"),
FPGA("--module-fpga"),
LIST_AS_USER(""), // no CLI switch supported yet.
- ADD_NUMA_PARAMS(""); // no CLI switch supported yet.
+ ADD_NUMA_PARAMS(""), // no CLI switch supported yet.
+ REMOVE_DOCKER_CONTAINER("--remove-docker-container"),
+ INSPECT_DOCKER_CONTAINER("--inspect-docker-container");
private final String option;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 9c05c59..ec1d055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor;
@@ -384,7 +385,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
Container container) throws ContainerExecutionException {
try {
String commandFile = dockerClient.writeCommandToTempFile(
- dockerVolumeCommand, container, nmContext);
+ dockerVolumeCommand, container.getContainerId(), nmContext);
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
privOp.appendArgs(commandFile);
@@ -734,6 +735,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
Container container = ctx.getContainer();
+ ContainerId containerId = container.getContainerId();
Map<String, String> environment = container.getLaunchContext()
.getEnvironment();
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
@@ -750,7 +752,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
validateImageName(imageName);
- String containerIdStr = container.getContainerId().toString();
+ String containerIdStr = containerId.toString();
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
String dockerRunAsUser = runAsUser;
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
@@ -908,7 +910,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String commandFile = dockerClient.writeCommandToTempFile(runCommand,
- container, nmContext);
+ containerId, nmContext);
PrivilegedOperation launchOp = buildLaunchOp(ctx,
commandFile, runCommand);
@@ -927,8 +929,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void relaunchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
- Container container = ctx.getContainer();
- String containerIdStr = container.getContainerId().toString();
+ ContainerId containerId = ctx.getContainer().getContainerId();
+ String containerIdStr = containerId.toString();
// Check to see if the container already exists for relaunch
DockerCommandExecutor.DockerContainerStatus containerStatus =
DockerCommandExecutor.getContainerStatus(containerIdStr, conf,
@@ -937,7 +939,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerCommandExecutor.isStartable(containerStatus)) {
DockerStartCommand startCommand = new DockerStartCommand(containerIdStr);
String commandFile = dockerClient.writeCommandToTempFile(startCommand,
- container, nmContext);
+ containerId, nmContext);
PrivilegedOperation launchOp = buildLaunchOp(ctx, commandFile,
startCommand);
@@ -1042,12 +1044,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
// ipAndHost[1] contains the hostname.
@Override
public String[] getIpAndHost(Container container) {
- String containerId = container.getContainerId().toString();
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
DockerInspectCommand inspectCommand =
- new DockerInspectCommand(containerId).getIpAndHost();
+ new DockerInspectCommand(containerIdStr).getIpAndHost();
try {
String commandFile = dockerClient.writeCommandToTempFile(inspectCommand,
- container, nmContext);
+ containerId, nmContext);
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
privOp.appendArgs(commandFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index c55b83b..7bd4546 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.slf4j.Logger;
@@ -104,9 +103,9 @@ public final class DockerClient {
}
}
- public String writeCommandToTempFile(DockerCommand cmd, Container container,
- Context nmContext) throws ContainerExecutionException {
- ContainerId containerId = container.getContainerId();
+ public String writeCommandToTempFile(DockerCommand cmd,
+ ContainerId containerId, Context nmContext)
+ throws ContainerExecutionException {
String filePrefix = containerId.toString();
ApplicationId appId = containerId.getApplicationAttemptId()
.getApplicationId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
index 0124c83..366457d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -22,7 +22,12 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import java.util.ArrayList;
import java.util.Collections;
@@ -104,4 +109,31 @@ public abstract class DockerCommand {
addCommandArguments("docker-config", clientConfigDir);
}
}
+
+ /**
+ * Prepare the privileged operation object that will be used to invoke
+ * the container-executor.
+ *
+ * @param dockerCommand Specific command to be run by docker.
+ * @param containerName
+ * @param env
+ * @param conf
+ * @param nmContext
+ * @return Returns the PrivilegedOperation object to be used.
+ * @throws ContainerExecutionException
+ */
+ public PrivilegedOperation preparePrivilegedOperation(
+ DockerCommand dockerCommand, String containerName, Map<String,
+ String> env, Configuration conf, Context nmContext)
+ throws ContainerExecutionException {
+ DockerClient dockerClient = new DockerClient(conf);
+ String commandFile =
+ dockerClient.writeCommandToTempFile(dockerCommand,
+ ContainerId.fromString(containerName),
+ nmContext);
+ PrivilegedOperation dockerOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
+ dockerOp.appendArgs(commandFile);
+ return dockerOp;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 6abe1cb..8a4888c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -80,14 +79,9 @@ public final class DockerCommandExecutor {
PrivilegedOperationExecutor privilegedOperationExecutor,
boolean disableFailureLogging, Context nmContext)
throws ContainerExecutionException {
- DockerClient dockerClient = new DockerClient(conf);
- String commandFile =
- dockerClient.writeCommandToTempFile(dockerCommand,
- nmContext.getContainers().get(ContainerId.fromString(containerId)),
- nmContext);
- PrivilegedOperation dockerOp = new PrivilegedOperation(
- PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
- dockerOp.appendArgs(commandFile);
+ PrivilegedOperation dockerOp = dockerCommand.preparePrivilegedOperation(
+ dockerCommand, containerId, env, conf, nmContext);
+
if (disableFailureLogging) {
dockerOp.disableFailureLogging();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index d27f74d0..3ed9c18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -20,12 +20,19 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+
+import java.util.Map;
+
/**
* Encapsulates the docker inspect command and its command
* line arguments.
*/
public class DockerInspectCommand extends DockerCommand {
private static final String INSPECT_COMMAND = "inspect";
+ private String commandArguments;
public DockerInspectCommand(String containerName) {
super(INSPECT_COMMAND);
@@ -34,6 +41,7 @@ public class DockerInspectCommand extends DockerCommand {
public DockerInspectCommand getContainerStatus() {
super.addCommandArguments("format", "{{.State.Status}}");
+ this.commandArguments = "--format={{.State.Status}}";
return this;
}
@@ -43,6 +51,17 @@ public class DockerInspectCommand extends DockerCommand {
// cannot parse the arguments correctly.
super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}"
+ "{{.IPAddress}},{{end}}{{.Config.Hostname}}");
+ this.commandArguments = "--format={{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}";
return this;
}
+ @Override
+ public PrivilegedOperation preparePrivilegedOperation(
+ DockerCommand dockerCommand, String containerName, Map<String,
+ String> env, Configuration conf, Context nmContext) {
+ PrivilegedOperation dockerOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.INSPECT_DOCKER_CONTAINER);
+ dockerOp.appendArgs(commandArguments, containerName);
+ return dockerOp;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index dcfe777..3a02982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -16,6 +16,12 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
+
+import java.util.Map;
+
/**
* Encapsulates the docker rm command and its command
* line arguments.
@@ -27,4 +33,14 @@ public class DockerRmCommand extends DockerCommand {
super(RM_COMMAND);
super.addCommandArguments("name", containerName);
}
+
+ @Override
+ public PrivilegedOperation preparePrivilegedOperation(
+ DockerCommand dockerCommand, String containerName, Map<String,
+ String> env, Configuration conf, Context nmContext) {
+ PrivilegedOperation dockerOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.REMOVE_DOCKER_CONTAINER);
+ dockerOp.appendArgs(containerName);
+ return dockerOp;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index d9ed070..6b4ec0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1332,6 +1332,34 @@ int run_docker(const char *command_file) {
return exit_code;
}
+int exec_docker_command(char *docker_command, char **argv,
+ int argc, int optind) {
+ int i;
+ char* docker_binary = get_docker_binary(&CFG);
+ size_t command_size = argc - optind + 2;
+
+ char **args = alloc_and_clear_memory(command_size + 1, sizeof(char));
+ args[0] = docker_binary;
+ args[1] = docker_command;
+ for(i = 2; i < command_size; i++) {
+ args[i] = (char *) argv[i];
+ }
+ args[i] = NULL;
+
+ execvp(docker_binary, args);
+
+ // will only get here if execvp fails
+ fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
+ docker_binary, strerror(errno));
+ fflush(LOGFILE);
+ fflush(ERRORFILE);
+
+ free(docker_binary);
+ free(args);
+
+ return DOCKER_RUN_FAILED;
+}
+
int create_script_paths(const char *work_dir,
const char *script_name, const char *cred_file,
char** script_file_dest, char** cred_file_dest,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 7c3ed77..47c4221 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -47,7 +47,9 @@ enum operations {
RUN_AS_USER_DELETE = 9,
RUN_AS_USER_LAUNCH_DOCKER_CONTAINER = 10,
RUN_DOCKER = 11,
- RUN_AS_USER_LIST = 12
+ RUN_AS_USER_LIST = 12,
+ REMOVE_DOCKER_CONTAINER = 13,
+ INSPECT_DOCKER_CONTAINER = 14
};
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -263,6 +265,12 @@ int is_docker_support_enabled();
*/
int run_docker(const char *command_file);
+/**
+ * Run a docker command without a command file
+ */
+int exec_docker_command(char *docker_command, char **argv,
+ int argc, int optind);
+
/*
* Compile the regex_str and determine if the input string matches.
* Return 0 on match, 1 of non-match.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index b69546a..c54fd3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -36,7 +36,7 @@ static void display_usage(FILE *stream) {
fprintf(stream,
"Usage: container-executor --checksetup\n"
" container-executor --mount-cgroups <hierarchy> "
- "<controller=path>...\n" );
+ "<controller=path>\n" );
if(is_tc_support_enabled()) {
fprintf(stream,
@@ -52,10 +52,15 @@ static void display_usage(FILE *stream) {
if(is_docker_support_enabled()) {
fprintf(stream,
- " container-executor --run-docker <command-file>\n");
+ " container-executor --run-docker <command-file>\n"
+ " container-executor --remove-docker-container <container_id>\n"
+ " container-executor --inspect-docker-container <container_id>\n");
} else {
fprintf(stream,
- "[DISABLED] container-executor --run-docker <command-file>\n");
+ "[DISABLED] container-executor --run-docker <command-file>\n"
+ "[DISABLED] container-executor --remove-docker-container <container_id>\n"
+ "[DISABLED] container-executor --inspect-docker-container "
+ "<format> ... <container_id>\n");
}
fprintf(stream,
@@ -331,6 +336,36 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
}
+ if (strcmp("--remove-docker-container", argv[1]) == 0) {
+ if(is_docker_support_enabled()) {
+ if (argc != 3) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ *operation = REMOVE_DOCKER_CONTAINER;
+ return 0;
+ } else {
+ display_feature_disabled_message("docker");
+ return FEATURE_DISABLED;
+ }
+ }
+
+ if (strcmp("--inspect-docker-container", argv[1]) == 0) {
+ if(is_docker_support_enabled()) {
+ if (argc != 4) {
+ display_usage(stdout);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ optind++;
+ *operation = INSPECT_DOCKER_CONTAINER;
+ return 0;
+ } else {
+ display_feature_disabled_message("docker");
+ return FEATURE_DISABLED;
+ }
+ }
+
/* Now we have to validate 'run as user' operations that don't use
a 'long option' - we should fix this at some point. The validation/argument
parsing here is extensive enough that it done in a separate function */
@@ -561,6 +596,12 @@ int main(int argc, char **argv) {
case RUN_DOCKER:
exit_code = run_docker(cmd_input.docker_command_file);
break;
+ case REMOVE_DOCKER_CONTAINER:
+ exit_code = exec_docker_command("rm", argv, argc, optind);
+ break;
+ case INSPECT_DOCKER_CONTAINER:
+ exit_code = exec_docker_command("inspect", argv, argc, optind);
+ break;
case RUN_AS_USER_INITIALIZE_CONTAINER:
exit_code = set_user(cmd_input.run_as_user_name);
if (exit_code != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/883f6822/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index a230d4d..50d00bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -153,14 +153,14 @@ public class TestDockerCommandExecutor {
env, configuration, mockExecutor, false, nmContext);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
- List<String> dockerCommands = getValidatedDockerCommands(ops);
+ PrivilegedOperation privOp = ops.get(0);
+ List<String> args = privOp.getArguments();
assertEquals(1, ops.size());
- assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
- ops.get(0).getOperationType().name());
- assertEquals(3, dockerCommands.size());
- assertEquals("[docker-command-execution]", dockerCommands.get(0));
- assertEquals(" docker-command=rm", dockerCommands.get(1));
- assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
+ assertEquals(PrivilegedOperation.OperationType.
+ REMOVE_DOCKER_CONTAINER.name(),
+ privOp.getOperationType().name());
+ assertEquals(1, args.size());
+ assertEquals(MOCK_CONTAINER_ID, args.get(0));
}
@Test
@@ -188,16 +188,15 @@ public class TestDockerCommandExecutor {
env, configuration, mockExecutor, false, nmContext);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
.capturePrivilegedOperations(mockExecutor, 1, true);
- List<String> dockerCommands = getValidatedDockerCommands(ops);
+ PrivilegedOperation privOp = ops.get(0);
+ List<String> args = privOp.getArguments();
assertEquals(1, ops.size());
- assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
- ops.get(0).getOperationType().name());
- assertEquals(4, dockerCommands.size());
- assertEquals("[docker-command-execution]", dockerCommands.get(0));
- assertEquals(" docker-command=inspect", dockerCommands.get(1));
- assertEquals(" format={{.State.Status}}", dockerCommands.get(2));
- assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(3));
-
+ assertEquals(PrivilegedOperation.OperationType.
+ INSPECT_DOCKER_CONTAINER.name(),
+ privOp.getOperationType().name());
+ assertEquals(2, args.size());
+ assertEquals("--format={{.State.Status}}", args.get(0));
+ assertEquals(MOCK_CONTAINER_ID, args.get(1));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org