You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/09/28 23:46:02 UTC
[4/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
YARN-6623. Add support to turn off launching privileged containers in the container-executor. (Varun Vasudev via wangda)
Change-Id: I76aec68cbfb42d239279d7245a03290d691e87a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3b1c631
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3b1c631
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3b1c631
Branch: refs/heads/trunk
Commit: d3b1c6319546706c41a2011ead6c3fe208883200
Parents: c114da5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Sep 28 16:41:09 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Sep 28 16:41:09 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn/conf/container-executor.cfg | 12 +
.../src/CMakeLists.txt | 5 +-
.../runtime/DockerLinuxContainerRuntime.java | 11 +-
.../linux/runtime/docker/DockerClient.java | 25 +-
.../linux/runtime/docker/DockerCommand.java | 55 +-
.../runtime/docker/DockerCommandExecutor.java | 3 +-
.../runtime/docker/DockerInspectCommand.java | 13 +-
.../linux/runtime/docker/DockerLoadCommand.java | 2 +-
.../linux/runtime/docker/DockerPullCommand.java | 2 +-
.../linux/runtime/docker/DockerRmCommand.java | 4 +-
.../linux/runtime/docker/DockerRunCommand.java | 68 +-
.../linux/runtime/docker/DockerStopCommand.java | 6 +-
.../container-executor/impl/configuration.c | 17 +
.../container-executor/impl/configuration.h | 19 +-
.../impl/container-executor.c | 316 +----
.../impl/container-executor.h | 9 -
.../container-executor/impl/get_executable.c | 3 -
.../container-executor/impl/get_executable.h | 29 +
.../main/native/container-executor/impl/main.c | 31 +-
.../impl/modules/common/module-configs.c | 3 +-
.../impl/modules/common/module-configs.h | 1 +
.../main/native/container-executor/impl/util.c | 60 +-
.../main/native/container-executor/impl/util.h | 46 +-
.../container-executor/impl/utils/docker-util.c | 998 ++++++++++++++++
.../container-executor/impl/utils/docker-util.h | 147 +++
.../impl/utils/string-utils.c | 1 -
.../docker-container-executor.cfg | 13 +
.../test/test-container-executor.c | 149 +--
.../native/container-executor/test/test_util.cc | 37 +-
.../test/utils/test-string-utils.cc | 37 +-
.../test/utils/test_docker_util.cc | 1122 ++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 398 ++++---
.../docker/TestDockerCommandExecutor.java | 35 +-
.../docker/TestDockerInspectCommand.java | 29 +-
.../runtime/docker/TestDockerLoadCommand.java | 9 +-
.../runtime/docker/TestDockerPullCommand.java | 8 +-
.../runtime/docker/TestDockerRmCommand.java | 8 +-
.../runtime/docker/TestDockerRunCommand.java | 25 +-
.../runtime/docker/TestDockerStopCommand.java | 15 +-
.../src/site/markdown/DockerContainers.md | 33 +-
40 files changed, 3067 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d68cee8..023654b 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -2,3 +2,15 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
min.user.id=1000#Prevent other super-users
allowed.system.users=##comma separated list of system users who CAN run applications
+feature.tc.enabled=0
+
+# The configs below deal with settings for Docker
+#[docker]
+# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default
+# docker.binary=/usr/bin/docker
+# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
+# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container
+# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
+# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
+# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
+# docker.privileged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 3d5b506..9d83bf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -132,6 +132,7 @@ add_library(container
main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
main/native/container-executor/impl/modules/common/module-configs.c
main/native/container-executor/impl/modules/gpu/gpu-module.c
+ main/native/container-executor/impl/utils/docker-util.c
)
add_executable(container-executor
@@ -156,6 +157,7 @@ output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
add_executable(cetest
+ main/native/container-executor/impl/get_executable.c
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
@@ -163,6 +165,7 @@ add_executable(cetest
main/native/container-executor/test/utils/test-path-utils.cc
main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
main/native/container-executor/test/modules/gpu/test-gpu-module.cc
- main/native/container-executor/test/test_util.cc)
+ main/native/container-executor/test/test_util.cc
+ main/native/container-executor/test/utils/test_docker_util.cc)
target_link_libraries(cetest gtest container)
output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index b8d9b0a..29c6fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -559,8 +559,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runCommand.setCapabilities(capabilities);
if(cgroupsRootDirectory != null) {
- runCommand.addMountLocation(cgroupsRootDirectory,
- cgroupsRootDirectory + ":ro", false);
+ runCommand.addReadOnlyMountLocation(cgroupsRootDirectory,
+ cgroupsRootDirectory, false);
}
List<String> allDirs = new ArrayList<>(containerLocalDirs);
@@ -584,7 +584,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String src = validateMount(dir[0], localizedResources);
String dst = dir[1];
- runCommand.addMountLocation(src, dst + ":ro", true);
+ runCommand.addReadOnlyMountLocation(src, dst, true);
}
}
}
@@ -626,7 +626,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
- LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
+ LOG.info("Docker command used: " + runCommand);
throw new ContainerExecutionException("Launch container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
@@ -757,8 +757,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp.appendArgs(tcCommandFile);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand
- .getCommandWithArguments());
+ LOG.debug("Launching container with cmd: " + runCommand);
}
return launchOp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index 536a22d..77c53a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -23,7 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +34,8 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.List;
+import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -68,10 +70,25 @@ public final class DockerClient {
TMP_FILE_SUFFIX, new
File(tmpDirPath));
- Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile),
- "UTF-8");
+ Writer writer = new OutputStreamWriter(
+ new FileOutputStream(dockerCommandFile), "UTF-8");
PrintWriter printWriter = new PrintWriter(writer);
- printWriter.print(cmd.getCommandWithArguments());
+ printWriter.println("[docker-command-execution]");
+ for (Map.Entry<String, List<String>> entry :
+ cmd.getDockerCommandWithArguments().entrySet()) {
+ if (entry.getKey().contains("=")) {
+ throw new ContainerExecutionException(
+ "'=' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ if (entry.getValue().contains("\n")) {
+ throw new ContainerExecutionException(
+ "'\\n' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ printWriter.println(" " + entry.getKey() + "=" + StringUtils
+ .join(",", entry.getValue()));
+ }
printWriter.close();
return dockerCommandFile.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
index 3b76a5c..7802209 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,32 +37,55 @@ import java.util.List;
* e.g 'run', 'load', 'inspect' etc.,
*/
-public abstract class DockerCommand {
+public abstract class DockerCommand {
private final String command;
- private final List<String> commandWithArguments;
+ private final Map<String, List<String>> commandArguments;
protected DockerCommand(String command) {
+ String dockerCommandKey = "docker-command";
this.command = command;
- this.commandWithArguments = new ArrayList<>();
- commandWithArguments.add(command);
+ this.commandArguments = new TreeMap<>();
+ commandArguments.put(dockerCommandKey, new ArrayList<>());
+ commandArguments.get(dockerCommandKey).add(command);
}
- /** Returns the docker sub-command string being used
- * e.g 'run'
+ /**
+ * Returns the docker sub-command string being used
+ * e.g 'run'.
*/
public final String getCommandOption() {
return this.command;
}
- /** Add command commandWithArguments - this method is only meant for use by
- * sub-classes
- * @param arguments to be added
+ /**
+ * Add command commandWithArguments - this method is only meant for use by
+ * sub-classes.
+ *
+ * @param key name of the key to be added
+ * @param value value of the key
*/
- protected final void addCommandArguments(String... arguments) {
- this.commandWithArguments.addAll(Arrays.asList(arguments));
+ protected final void addCommandArguments(String key, String value) {
+ List<String> list = commandArguments.get(key);
+ if (list != null) {
+ list.add(value);
+ return;
+ }
+ list = new ArrayList<>();
+ list.add(value);
+ this.commandArguments.put(key, list);
}
- public String getCommandWithArguments() {
- return StringUtils.join(" ", commandWithArguments);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return Collections.unmodifiableMap(commandArguments);
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ StringBuffer ret = new StringBuffer(this.command);
+ for (Map.Entry<String, List<String>> entry : commandArguments.entrySet()) {
+ ret.append(" ").append(entry.getKey());
+ ret.append("=").append(StringUtils.join(",", entry.getValue()));
+ }
+ return ret.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 5739912..76b53af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -88,8 +88,7 @@ public final class DockerCommandExecutor {
dockerOp.disableFailureLogging();
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Running docker command: "
- + dockerCommand.getCommandWithArguments());
+ LOG.debug("Running docker command: " + dockerCommand);
}
try {
String result = privilegedOperationExecutor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index 812a35f..d27f74d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -26,16 +26,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
*/
public class DockerInspectCommand extends DockerCommand {
private static final String INSPECT_COMMAND = "inspect";
- private String containerName;
public DockerInspectCommand(String containerName) {
super(INSPECT_COMMAND);
- this.containerName = containerName;
+ super.addCommandArguments("name", containerName);
}
public DockerInspectCommand getContainerStatus() {
- super.addCommandArguments("--format='{{.State.Status}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{.State.Status}}");
return this;
}
@@ -43,9 +41,8 @@ public class DockerInspectCommand extends DockerCommand {
// Be sure to not use space in the argument, otherwise the
// extract_values_delim method in container-executor binary
// cannot parse the arguments correctly.
- super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}"
- + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}");
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
index e4d92e0..fa2988c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
@@ -25,6 +25,6 @@ public class DockerLoadCommand extends DockerCommand {
public DockerLoadCommand(String localImageFile) {
super(LOAD_COMMAND);
- super.addCommandArguments("--i=" + localImageFile);
+ super.addCommandArguments("image", localImageFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
index 351e09e..5e6108e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
@@ -25,7 +25,7 @@ public class DockerPullCommand extends DockerCommand {
public DockerPullCommand(String imageName) {
super(PULL_COMMAND);
- super.addCommandArguments(imageName);
+ super.addCommandArguments("image", imageName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index b1aea61..dcfe777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -25,6 +25,6 @@ public class DockerRmCommand extends DockerCommand {
public DockerRmCommand(String containerName) {
super(RM_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 1e1e6e8..c7bf827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -20,42 +20,39 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.hadoop.util.StringUtils;
-
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class DockerRunCommand extends DockerCommand {
private static final String RUN_COMMAND = "run";
- private final String image;
- private List<String> overrrideCommandWithArgs;
/** The following are mandatory: */
public DockerRunCommand(String containerId, String user, String image) {
super(RUN_COMMAND);
- super.addCommandArguments("--name=" + containerId, "--user=" + user);
- this.image = image;
+ super.addCommandArguments("name", containerId);
+ super.addCommandArguments("user", user);
+ super.addCommandArguments("image", image);
}
public DockerRunCommand removeContainerOnExit() {
- super.addCommandArguments("--rm");
+ super.addCommandArguments("rm", "true");
return this;
}
public DockerRunCommand detachOnRun() {
- super.addCommandArguments("-d");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand setContainerWorkDir(String workdir) {
- super.addCommandArguments("--workdir=" + workdir);
+ super.addCommandArguments("workdir", workdir);
return this;
}
public DockerRunCommand setNetworkType(String type) {
- super.addCommandArguments("--net=" + type);
+ super.addCommandArguments("net", type);
return this;
}
@@ -65,79 +62,80 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+ super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
+ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
+ destinationPath, boolean createSource) {
+ boolean sourceExists = new File(sourcePath).exists();
+ if (!sourceExists && !createSource) {
+ return this;
+ }
+ super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand setCGroupParent(String parentPath) {
- super.addCommandArguments("--cgroup-parent=" + parentPath);
+ super.addCommandArguments("cgroup-parent", parentPath);
return this;
}
/* Run a privileged container. Use with extreme care */
public DockerRunCommand setPrivileged() {
- super.addCommandArguments("--privileged");
+ super.addCommandArguments("privileged", "true");
return this;
}
public DockerRunCommand setCapabilities(Set<String> capabilties) {
//first, drop all capabilities
- super.addCommandArguments("--cap-drop=ALL");
+ super.addCommandArguments("cap-drop", "ALL");
//now, add the capabilities supplied
for (String capability : capabilties) {
- super.addCommandArguments("--cap-add=" + capability);
+ super.addCommandArguments("cap-add", capability);
}
return this;
}
public DockerRunCommand setHostname(String hostname) {
- super.addCommandArguments("--hostname=" + hostname);
+ super.addCommandArguments("hostname", hostname);
return this;
}
public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) {
- super.addCommandArguments("--device=" + sourceDevice + ":" +
+ super.addCommandArguments("devices", sourceDevice + ":" +
destinationDevice);
return this;
}
public DockerRunCommand enableDetach() {
- super.addCommandArguments("--detach=true");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand disableDetach() {
- super.addCommandArguments("--detach=false");
+ super.addCommandArguments("detach", "false");
return this;
}
public DockerRunCommand groupAdd(String[] groups) {
- for(int i = 0; i < groups.length; i++) {
- super.addCommandArguments("--group-add " + groups[i]);
- }
+ super.addCommandArguments("group-add", String.join(",", groups));
return this;
}
public DockerRunCommand setOverrideCommandWithArgs(
List<String> overrideCommandWithArgs) {
- this.overrrideCommandWithArgs = overrideCommandWithArgs;
+ for(String override: overrideCommandWithArgs) {
+ super.addCommandArguments("launch-command", override);
+ }
return this;
}
@Override
- public String getCommandWithArguments() {
- List<String> argList = new ArrayList<>();
-
- argList.add(super.getCommandWithArguments());
- argList.add(image);
-
- if (overrrideCommandWithArgs != null) {
- argList.addAll(overrrideCommandWithArgs);
- }
-
- return StringUtils.join(" ", argList);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return super.getDockerCommandWithArguments();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
index e9d6c43..ccfff82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
@@ -29,11 +29,11 @@ public class DockerStopCommand extends DockerCommand {
public DockerStopCommand(String containerName) {
super(STOP_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
public DockerStopCommand setGracePeriod(int value) {
- super.addCommandArguments("--time=" + Integer.toString(value));
+ super.addCommandArguments("time", Integer.toString(value));
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 12dbc4c..f23cff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -21,6 +21,7 @@
#include "configuration.h"
#include "util.h"
+#include "get_executable.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
@@ -696,3 +697,19 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
return 0;
}
+
+char *get_config_path(const char *argv0) {
+ char *executable_file = get_executable((char *) argv0);
+ if (!executable_file) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
+ return NULL;
+ }
+
+ const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ if (conf_file == NULL) {
+ fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
+ }
+ return conf_file;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 1ea5561..7fa684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -23,10 +23,21 @@
#define _WITH_GETLINE
#endif
-#include <stddef.h>
+#include "config.h"
+
+#define CONF_FILENAME "container-executor.cfg"
+
+// When building as part of a Maven build this value gets defined by using
+// container-executor.conf.dir property. See:
+// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+// for details.
+// NOTE: if this ends up being a relative path it gets resolved relative to
+// the location of the container-executor binary itself, not getwd(3)
+#ifndef HADOOP_CONF_DIR
+#error HADOOP_CONF_DIR must be defined
+#endif
-/** Define a platform-independent constant instead of using PATH_MAX */
-#define EXECUTOR_PATH_MAX 4096
+#include <stddef.h>
// Configuration data structures.
struct kv_pair {
@@ -207,4 +218,6 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
+char *get_config_path(const char* argv0);
+
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index e8bf564..08d69a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,7 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
-#include "utils/string-utils.h"
+#include "utils/docker-util.h"
#include "util.h"
#include "config.h"
@@ -43,7 +43,6 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
-#include <regex.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -81,11 +80,6 @@ static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
//struct to store the user details
struct passwd *user_detail = NULL;
-//Docker container related constants.
-static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
-static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
-static const char* DOCKER_PULL_COMMAND = "pull";
-
FILE* LOGFILE = NULL;
FILE* ERRORFILE = NULL;
@@ -465,8 +459,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
}
int is_docker_support_enabled() {
- return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
- DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg);
+ return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
+ DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg)
+ || docker_module_enabled(&CFG);
}
int is_tc_support_enabled() {
@@ -474,13 +469,6 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
-char* check_docker_binary(char *docker_binary) {
- if (docker_binary == NULL) {
- return "docker";
- }
- return docker_binary;
-}
-
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -1157,273 +1145,28 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
-static char* escape_single_quote(const char *str) {
- int p = 0;
- int i = 0;
- char replacement[] = "'\"'\"'";
- size_t replacement_length = strlen(replacement);
- size_t ret_size = strlen(str) * replacement_length + 1;
- char *ret = (char *) calloc(ret_size, sizeof(char));
- if(ret == NULL) {
- exit(OUT_OF_MEMORY);
- }
- while(str[p] != '\0') {
- if(str[p] == '\'') {
- strncat(ret, replacement, ret_size - strlen(ret));
- i += replacement_length;
- }
- else {
- ret[i] = str[p];
- ret[i + 1] = '\0';
- i++;
- }
- p++;
- }
- return ret;
-}
-
-static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
- char *tmp = escape_single_quote(arg);
- strcat(*str, param);
- strcat(*str, "'");
- if(strlen(*str) + strlen(tmp) > *size) {
- *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
- if(*str == NULL) {
- exit(OUT_OF_MEMORY);
- }
- *size = strlen(*str) + strlen(tmp) + 1024;
- }
- strcat(*str, tmp);
- strcat(*str, "' ");
- free(tmp);
-}
-
-char** tokenize_docker_command(const char *input, int *split_counter) {
- char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
- char **linesplit = (char **) malloc(sizeof(char *));
- char *p = NULL;
- *split_counter = 0;
- strncpy(line, input, strlen(input));
-
- p = strtok(line, " ");
- while(p != NULL) {
- linesplit[*split_counter] = p;
- (*split_counter)++;
- linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
- if(linesplit == NULL) {
- fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
- strerror(errno));
- fflush(ERRORFILE);
- exit(OUT_OF_MEMORY);
- }
- p = strtok(NULL, " ");
- }
- linesplit[*split_counter] = NULL;
- return linesplit;
-}
-
-int execute_regex_match(const char *regex_str, const char *input) {
- regex_t regex;
- int regex_match;
- if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
- fprintf(LOGFILE, "Unable to compile regex.");
- fflush(LOGFILE);
- exit(ERROR_COMPILING_REGEX);
- }
- regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
- regfree(®ex);
- if(0 == regex_match) {
- return 0;
- }
- return 1;
-}
-
-int validate_docker_image_name(const char *image_name) {
- char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
- return execute_regex_match(regex_str, image_name);
-}
-
-char* sanitize_docker_command(const char *line) {
- static struct option long_options[] = {
- {"name", required_argument, 0, 'n' },
- {"user", required_argument, 0, 'u' },
- {"rm", no_argument, 0, 'r' },
- {"workdir", required_argument, 0, 'w' },
- {"net", required_argument, 0, 'e' },
- {"hostname", required_argument, 0, 'h' },
- {"cgroup-parent", required_argument, 0, 'g' },
- {"privileged", no_argument, 0, 'p' },
- {"cap-add", required_argument, 0, 'a' },
- {"cap-drop", required_argument, 0, 'o' },
- {"device", required_argument, 0, 'i' },
- {"detach", required_argument, 0, 't' },
- {"format", required_argument, 0, 'f' },
- {"group-add", required_argument, 0, 'x' },
- {0, 0, 0, 0}
- };
-
- int c = 0;
- int option_index = 0;
- char *output = NULL;
- size_t output_size = 0;
- char **linesplit;
- int split_counter = 0;
- int len = strlen(line);
-
- linesplit = tokenize_docker_command(line, &split_counter);
-
- output_size = len * 2;
- output = (char *) calloc(output_size, sizeof(char));
- if(output == NULL) {
- exit(OUT_OF_MEMORY);
- }
-
- // Handle docker client config option.
- if(0 == strncmp(linesplit[0], DOCKER_CLIENT_CONFIG_ARG, strlen(DOCKER_CLIENT_CONFIG_ARG))) {
- strcat(output, linesplit[0]);
- strcat(output, " ");
- long index = 0;
- while(index < split_counter) {
- linesplit[index] = linesplit[index + 1];
- if (linesplit[index] == NULL) {
- split_counter--;
- break;
- }
- index++;
- }
- }
-
- // Handle docker pull and image name validation.
- if (0 == strncmp(linesplit[0], DOCKER_PULL_COMMAND, strlen(DOCKER_PULL_COMMAND))) {
- if (0 != validate_docker_image_name(linesplit[1])) {
- fprintf(ERRORFILE, "Invalid Docker image name, exiting.");
- fflush(ERRORFILE);
- exit(DOCKER_IMAGE_INVALID);
- }
- strcat(output, linesplit[0]);
- strcat(output, " ");
- strcat(output, linesplit[1]);
- return output;
- }
-
- strcat(output, linesplit[0]);
- strcat(output, " ");
- optind = 1;
- while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
- switch(c) {
- case 'n':
- quote_and_append_arg(&output, &output_size, "--name=", optarg);
- break;
- case 'w':
- quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
- break;
- case 'u':
- quote_and_append_arg(&output, &output_size, "--user=", optarg);
- break;
- case 'e':
- quote_and_append_arg(&output, &output_size, "--net=", optarg);
- break;
- case 'h':
- quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
- break;
- case 'v':
- quote_and_append_arg(&output, &output_size, "-v ", optarg);
- break;
- case 'a':
- quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
- break;
- case 'o':
- quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
- break;
- case 'd':
- strcat(output, "-d ");
- break;
- case 'r':
- strcat(output, "--rm ");
- break;
- case 'g':
- quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
- break;
- case 'p':
- strcat(output, "--privileged ");
- break;
- case 'i':
- quote_and_append_arg(&output, &output_size, "--device=", optarg);
- break;
- case 't':
- quote_and_append_arg(&output, &output_size, "--detach=", optarg);
- break;
- case 'f':
- strcat(output, "--format=");
- strcat(output, optarg);
- strcat(output, " ");
- break;
- case 'x':
- quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
- break;
- default:
- fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
- fflush(LOGFILE);
- return NULL;
- break;
- }
- }
-
- if(optind < split_counter) {
- while(optind < split_counter) {
- if (0 == strncmp(linesplit[optind], DOCKER_CONTAINER_NAME_PREFIX, strlen(DOCKER_CONTAINER_NAME_PREFIX))) {
- if (1 != validate_container_id(linesplit[optind])) {
- fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", linesplit[optind]);
- fflush(ERRORFILE);
- exit(DOCKER_CONTAINER_NAME_INVALID);
- }
- strcat(output, linesplit[optind++]);
- } else {
- quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
- }
- }
- }
-
- return output;
-}
-
-char* parse_docker_command_file(const char* command_file) {
- size_t len = 0;
- char *line = NULL;
- ssize_t read;
- FILE *stream;
- stream = fopen(command_file, "r");
- if (stream == NULL) {
- fprintf(ERRORFILE, "Cannot open file %s - %s",
- command_file, strerror(errno));
- fflush(ERRORFILE);
- exit(ERROR_OPENING_DOCKER_FILE);
- }
- if ((read = getline(&line, &len, stream)) == -1) {
- fprintf(ERRORFILE, "Error reading command_file %s\n", command_file);
- fflush(ERRORFILE);
- exit(ERROR_READING_DOCKER_FILE);
- }
- fclose(stream);
-
- char* ret = sanitize_docker_command(line);
- if(ret == NULL) {
- exit(ERROR_SANITIZING_DOCKER_COMMAND);
+char *construct_docker_command(const char *command_file) {
+ int ret = 0;
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
+ char *buffer = alloc_and_clear_memory(command_size, sizeof(char));
+ ret = get_docker_command(command_file, &CFG, buffer, command_size);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Error constructing docker command, docker error code=%d, error message='%s'\n", ret,
+ get_docker_error_message(ret));
+ fflush(ERRORFILE);
+ exit(DOCKER_RUN_FAILED);
}
- fprintf(ERRORFILE, "Using command %s\n", ret);
- fflush(ERRORFILE);
-
- return ret;
+ return buffer;
}
int run_docker(const char *command_file) {
- char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char* docker_command = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- char* docker_command_with_binary = calloc(sizeof(char), command_size);
+ char* docker_command_with_binary = alloc_and_clear_memory(command_size, sizeof(char));
snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
+ fprintf(LOGFILE, "Invoking '%s'\n", docker_command_with_binary);
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
@@ -1437,8 +1180,9 @@ int run_docker(const char *command_file) {
free(docker_command_with_binary);
free(docker_command);
exit_code = DOCKER_RUN_FAILED;
+ } else {
+ exit_code = 0;
}
- exit_code = 0;
return exit_code;
}
@@ -1583,18 +1327,17 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- docker_command_with_binary = calloc(sizeof(char), command_size);
- docker_wait_command = calloc(sizeof(char), command_size);
- docker_logs_command = calloc(sizeof(char), command_size);
- docker_inspect_command = calloc(sizeof(char), command_size);
- docker_rm_command = calloc(sizeof(char), command_size);
+ docker_command_with_binary = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_wait_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_logs_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_inspect_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_rm_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
gid_t user_gid = getegid();
uid_t prev_uid = geteuid();
- char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char *docker_command = NULL;
+ char *docker_binary = NULL;
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
@@ -1617,6 +1360,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
+ docker_command = construct_docker_command(command_file);
+ docker_binary = get_docker_binary(&CFG);
+
fprintf(LOGFILE, "Getting exit code file...\n");
exit_code_file = get_exit_code_file(pid_file);
if (NULL == exit_code_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index aa38abf..6d4220f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,16 +59,12 @@ enum operations {
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
-#define DOCKER_BINARY_KEY "docker.binary"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
-// get the executable's filename
-char* get_executable(char *argv0);
-
//function used to load the configurations present in the secure config
void read_executor_config(const char* file_name);
@@ -258,11 +254,6 @@ int is_docker_support_enabled();
*/
int run_docker(const char *command_file);
-/**
- * Sanitize docker commands. Returns NULL if there was any failure.
-*/
-char* sanitize_docker_command(const char *line);
-
/*
* Compile the regex_str and determine if the input string matches.
* Return 0 on match, 1 of non-match.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index 55973a2..e1ec293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -29,12 +29,9 @@
*/
#include "config.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "util.h"
#include <errno.h>
-#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
new file mode 100644
index 0000000..ee6b375
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+
+/**
+ * Get the path to executable that is currently running
+ * @param argv0 the name of the executable
+ * @return the path to the currently running executable
+ */
+char* get_executable(char *argv0);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index a05dc78..7b8f63f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -20,30 +20,15 @@
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
+#include "get_executable.h"
#include "modules/gpu/gpu-module.h"
#include "modules/cgroups/cgroups-operations.h"
#include <errno.h>
#include <grp.h>
-#include <limits.h>
#include <unistd.h>
-#include <signal.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/stat.h>
-
-#define CONF_FILENAME "container-executor.cfg"
-
-// When building as part of a Maven build this value gets defined by using
-// container-executor.conf.dir property. See:
-// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
-// for details.
-// NOTE: if this ends up being a relative path it gets resolved relative to
-// the location of the container-executor binary itself, not getwd(3)
-#ifndef HADOOP_CONF_DIR
- #error HADOOP_CONF_DIR must be defined
-#endif
static void display_usage(FILE *stream) {
fprintf(stream,
@@ -145,25 +130,21 @@ of whether an explicit checksetup operation is requested. */
static void assert_valid_setup(char *argv0) {
int ret;
char *executable_file = get_executable(argv0);
- if (!executable_file) {
- fprintf(ERRORFILE,"realpath of executable: %s\n",
- errno != 0 ? strerror(errno) : "unknown");
+ if (!executable_file || executable_file[0] == 0) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
flush_and_close_log_files();
- exit(-1);
+ exit(INVALID_CONFIG_FILE);
}
- char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
- char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ char *conf_file = get_config_path(argv0);
if (conf_file == NULL) {
- free(executable_file);
- fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
- free(executable_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
index f0c6d16..4c76e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
@@ -16,9 +16,8 @@
* limitations under the License.
*/
+#include "module-configs.h"
#include "util.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "modules/common/constants.h"
#include <string.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
index d58c618..b8c57df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
@@ -23,6 +23,7 @@
#ifndef _MODULES_COMMON_MODULE_CONFIGS_H_
#define _MODULES_COMMON_MODULE_CONFIGS_H_
+#include "configuration.h"
/**
* check if module enabled given name of module.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
index 8e39ca8..a9539cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -17,10 +17,10 @@
*/
#include "util.h"
-#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
+#include <regex.h>
char** split_delimiter(char *value, const char *delim) {
char **return_values = NULL;
@@ -132,3 +132,61 @@ char* trim(const char* input) {
ret[val_end - val_begin] = '\0';
return ret;
}
+
+int execute_regex_match(const char *regex_str, const char *input) {
+ regex_t regex;
+ int regex_match;
+ if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
+ fprintf(LOGFILE, "Unable to compile regex.");
+ fflush(LOGFILE);
+ exit(ERROR_COMPILING_REGEX);
+ }
+ regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
+ regfree(®ex);
+ if(0 == regex_match) {
+ return 0;
+ }
+ return 1;
+}
+
+char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) alloc_and_clear_memory(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ i++;
+ }
+ p++;
+ }
+ ret[i] = '\0';
+ return ret;
+}
+
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ int alloc_block = 1024;
+ strcat(*str, param);
+ strcat(*str, "'");
+ if (strlen(*str) + strlen(tmp) > *size) {
+ *size = (strlen(*str) + strlen(tmp) + alloc_block) * sizeof(char);
+ *str = (char *) realloc(*str, *size);
+ if (*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index fa21def..8758d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -19,7 +19,11 @@
#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+/** Define a platform-independent constant instead of using PATH_MAX, set to 4K */
+#define EXECUTOR_PATH_MAX 4096
+
#include <stdio.h>
+#include <stdlib.h>
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
@@ -62,7 +66,7 @@ enum errorcodes {
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
- DOCKER_CONTAINER_NAME_INVALID = 41,
+ // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42
};
@@ -119,4 +123,44 @@ void free_values(char **values);
*/
char* trim(const char *input);
+/**
+ * Run a regex to check if the provided input matches against it
+ * @param regex_str Regex to run
+ * @param input String to run the regex against
+ * @return 0 on match, non-zero on no match
+ */
+int execute_regex_match(const char *regex_str, const char *input);
+
+/**
+ * Helper function to escape single-quotes in a string. The assumption is that the string passed will be enclosed in
+ * single quotes and passed to bash for a command invocation.
+ * @param str The string in which to esacpe single quotes
+ * @return String with single quotes escaped, must be freed by the user.
+ */
+char* escape_single_quote(const char *str);
+
+/**
+ * Helper function to quote the argument to a parameter and then append it to the provided string.
+ * @param str Buffer to which the param='arg' string must be appended
+ * @param size Size of the buffer
+ * @param param Param name
+ * @param arg Argument to be quoted
+ */
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg);
+
+/**
+ * Helper function to allocate and clear a block of memory. It'll call exit if the allocation fails.
+ * @param num Num of elements to be allocated
+ * @param size Size of each element
+ * @return Pointer to the allocated memory, must be freed by the user
+ */
+inline void* alloc_and_clear_memory(size_t num, size_t size) {
+ void *ret = calloc(num, size);
+ if (ret == NULL) {
+ printf("Could not allocate memory, exiting\n");
+ exit(OUT_OF_MEMORY);
+ }
+ return ret;
+}
+
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org