You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/09/28 23:45:59 UTC
[1/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk c114da5e6 -> d3b1c6319
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index 60fce40..05b44b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -114,8 +114,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("rm " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=rm", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -130,8 +132,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("stop " + MOCK_CONTAINER_ID, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=stop", dockerCommands.get(1));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(2));
}
@Test
@@ -147,9 +151,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("inspect --format='{{.State.Status}}' " + MOCK_CONTAINER_ID,
- dockerCommands.get(0));
+ assertEquals(4, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=inspect", dockerCommands.get(1));
+ assertEquals(" format={{.State.Status}}", dockerCommands.get(2));
+ assertEquals(" name=" + MOCK_CONTAINER_ID, dockerCommands.get(3));
+
}
@Test
@@ -165,8 +172,10 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("pull " + MOCK_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=pull", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_IMAGE_NAME, dockerCommands.get(2));
}
@Test
@@ -182,8 +191,12 @@ public class TestDockerCommandExecutor {
assertEquals(1, ops.size());
assertEquals(PrivilegedOperation.OperationType.RUN_DOCKER_CMD.name(),
ops.get(0).getOperationType().name());
- assertEquals(1, dockerCommands.size());
- assertEquals("load --i=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(0));
+ assertEquals(3, dockerCommands.size());
+ assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ assertEquals(" docker-command=load", dockerCommands.get(1));
+ assertEquals(" image=" + MOCK_LOCAL_IMAGE_NAME, dockerCommands.get(2));
+
+
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
index 619f202..4092e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -44,16 +46,29 @@ public class TestDockerInspectCommand {
@Test
public void testGetContainerStatus() throws Exception {
dockerInspectCommand.getContainerStatus();
- assertEquals("inspect --format='{{.State.Status}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{.State.Status}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
@Test
public void testGetIpAndHost() throws Exception {
dockerInspectCommand.getIpAndHost();
- assertEquals(
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}}"
- + ",{{end}}{{.Config.Hostname}}' foo",
- dockerInspectCommand.getCommandWithArguments());
+ assertEquals("inspect", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("format")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerInspectCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(3,
+ dockerInspectCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
index 85fa0f8..e5bff26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerLoadCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,11 @@ public class TestDockerLoadCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("load --i=foo",
- dockerLoadCommand.getCommandWithArguments());
+ assertEquals("load", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerLoadCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerLoadCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
index 89157ff..ccf7000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +43,12 @@ public class TestDockerPullCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("pull foo", dockerPullCommand.getCommandWithArguments());
+ assertEquals("pull", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerPullCommand.getDockerCommandWithArguments().get("image")));
+ assertEquals(2, dockerPullCommand.getDockerCommandWithArguments().size());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
index d1b9904..a8d4bdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +44,11 @@ public class TestDockerRmCommand {
@Test
public void testGetCommandWithArguments() {
- assertEquals("rm foo", dockerRmCommand.getCommandWithArguments());
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
index 85bccd2..e51d7ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRunCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -56,8 +57,24 @@ public class TestDockerRunCommand {
commands.add("launch_command");
dockerRunCommand.setOverrideCommandWithArgs(commands);
dockerRunCommand.removeContainerOnExit();
- assertEquals("run --name=foo --user=user_id --device=source:dest --rm "
- + "image_name launch_command",
- dockerRunCommand.getCommandWithArguments());
+
+ assertEquals("run", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("user_id", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("user")));
+ assertEquals("image_name", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("image")));
+
+ assertEquals("source:dest", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments().get("devices")));
+ assertEquals("true", StringUtils
+ .join(",", dockerRunCommand.getDockerCommandWithArguments().get("rm")));
+ assertEquals("launch_command", StringUtils.join(",",
+ dockerRunCommand.getDockerCommandWithArguments()
+ .get("launch-command")));
+ assertEquals(7, dockerRunCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
index c9743f3..efbde77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerStopCommand.java
@@ -21,6 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
@@ -48,8 +50,13 @@ public class TestDockerStopCommand {
@Test
public void testSetGracePeriod() throws Exception {
dockerStopCommand.setGracePeriod(GRACE_PERIOD);
- assertEquals("stop foo --time=10",
- dockerStopCommand.getCommandWithArguments());
-
+ assertEquals("stop", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("name")));
+ assertEquals("10", StringUtils.join(",",
+ dockerStopCommand.getDockerCommandWithArguments().get("time")));
+ assertEquals(3, dockerStopCommand.getDockerCommandWithArguments().size());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 23f4134..36c391a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -167,7 +167,24 @@ The following properties are required to enable Docker support:
|Configuration Name | Description |
|:---- |:---- |
| `yarn.nodemanager.linux-container-executor.group` | The Unix group of the NodeManager. It should match the yarn.nodemanager.linux-container-executor.group in the yarn-site.xml file. |
-| `feature.docker.enabled` | Must be 0 or 1. 0 means launching Docker containers is disabled. 1 means launching Docker containers is allowed. |
+
+The container-executor.cfg must contain a section to determine the capabilities that containers
+are allowed. It contains the following properties:
+
+|Configuration Name | Description |
+|:---- |:---- |
+| `module.enabled` | Must be "true" or "false" to enable or disable launching Docker containers respectively. Default value is 0. |
+| `docker.binary` | The binary used to launch Docker containers. /usr/bin/docker by default. |
+| `docker.allowed.capabilities` | Comma separated capabilities that containers are allowed to add. By default no capabilities are allowed to be added. |
+| `docker.allowed.devices` | Comma separated devices that containers are allowed to mount. By default no devices are allowed to be added. |
+| `docker.allowed.networks` | Comma separated networks that containers are allowed to use. If no network is specified when launching the container, the default Docker network will be used. |
+| `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. |
+| `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. |
+| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. |
+
+Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.
+
+In addition, containers are not permitted to mount any parent of the container-executor.cfg directory in read-write mode.
The following properties are optional:
@@ -176,9 +193,21 @@ The following properties are optional:
| `min.user.id` | The minimum UID that is allowed to launch applications. The default is no minimum |
| `banned.users` | A comma-separated list of usernames who should not be allowed to launch applications. The default setting is: yarn, mapred, hdfs, and bin. |
| `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users and banned.users, the user will be considered banned. |
-| `docker.binary` | The path to the Docker binary. The default is "docker". |
| `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1 means traffic control commands are allowed. |
+Part of a container-executor.cfg which allows Docker containers to be launched is below:
+
+```
+yarn.nodemanager.linux-container-executor.group=yarn
+[docker]
+ module.enabled=true
+ docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE
+ docker.allowed.networks=bridge,host,none
+ docker.allowed.ro-mounts=/sys/fs/cgroup
+ docker.allowed.rw-mounts=/var/hadoop/yarn/local-dir,/var/hadoop/yarn/log-dir
+
+```
+
Docker Image Requirements
-------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
new file mode 100644
index 0000000..860320d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -0,0 +1,998 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <ctype.h>
+#include "../modules/common/module-configs.h"
+#include "docker-util.h"
+#include "string-utils.h"
+#include "util.h"
+
+static int read_and_verify_command_file(const char *command_file, const char *docker_command,
+ struct configuration *command_config) {
+ int ret = 0;
+ ret = read_config(command_file, command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (command == NULL || (strcmp(command, docker_command) != 0)) {
+ ret = INCORRECT_COMMAND;
+ }
+ free(command);
+ return ret;
+}
+
+static int add_to_buffer(char *buff, const size_t bufflen, const char *string) {
+ size_t current_len = strlen(buff);
+ size_t string_len = strlen(string);
+ if (current_len + string_len < bufflen - 1) {
+ strncpy(buff + current_len, string, string_len);
+ buff[current_len + string_len] = '\0';
+ return 0;
+ }
+ return -1;
+}
+
+static int add_param_to_command(const struct configuration *command_config, const char *key, const char *param,
+ const int with_argument, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ int ret = 0;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ if (value != NULL) {
+ if (with_argument) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, value);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ } else if (strcmp(value, "true") == 0) {
+ ret = add_to_buffer(out, outlen, param);
+ }
+ free(value);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ }
+ free(tmp_buffer);
+ return ret;
+}
+
+static int add_param_to_command_if_allowed(const struct configuration *command_config,
+ const struct configuration *executor_cfg,
+ const char *key, const char *allowed_key, const char *param,
+ const int multiple_values, const char prefix,
+ char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *tmp_ptr = NULL;
+ char **values = NULL;
+ char **permitted_values = get_configuration_values_delimiter(allowed_key,
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, executor_cfg,
+ ",");
+ int i = 0, j = 0, permitted = 0, ret = 0;
+ if (multiple_values) {
+ values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ } else {
+ values = (char **) alloc_and_clear_memory(2, sizeof(char *));
+ values[0] = get_configuration_value(key, DOCKER_COMMAND_FILE_SECTION, command_config);
+ values[1] = NULL;
+ if (values[0] == NULL) {
+ ret = 0;
+ goto free_and_exit;
+ }
+ }
+
+ if (values != NULL) {
+ if (permitted_values != NULL) {
+ for (i = 0; values[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ permitted = 0;
+ if(prefix != 0) {
+ tmp_ptr = strchr(values[i], prefix);
+ if (tmp_ptr == NULL) {
+ fprintf(ERRORFILE, "Prefix char '%c' not found in '%s'\n",
+ prefix, values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ for (j = 0; permitted_values[j] != NULL; ++j) {
+ if (prefix == 0) {
+ ret = strcmp(values[i], permitted_values[j]);
+ } else {
+ ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
+ }
+ if (ret == 0) {
+ permitted = 1;
+ break;
+ }
+ }
+ if (permitted == 1) {
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, param, values[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Output buffer too small\n");
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested\n", values[i]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+ } else {
+ fprintf(ERRORFILE, "Invalid param '%s' requested, "
+ "permitted values list is empty\n", values[0]);
+ ret = -1;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free_values(values);
+ free_values(permitted_values);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_docker_config_param(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "docker-config", "--config=", 1, out, outlen);
+}
+
+static int validate_container_name(const char *container_name) {
+ const char *CONTAINER_NAME_PREFIX = "container_";
+ if (0 == strncmp(container_name, CONTAINER_NAME_PREFIX, strlen(CONTAINER_NAME_PREFIX))) {
+ if (1 == validate_container_id(container_name)) {
+ return 0;
+ }
+ }
+ fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", container_name);
+ fflush(ERRORFILE);
+ return INVALID_DOCKER_CONTAINER_NAME;
+}
+
+const char *get_docker_error_message(const int error_code) {
+
+ switch (error_code) {
+ case INVALID_COMMAND_FILE:
+ return "Invalid command file passed";
+ case INCORRECT_COMMAND:
+ return "Incorrect command";
+ case BUFFER_TOO_SMALL:
+ return "Command buffer too small";
+ case INVALID_DOCKER_CONTAINER_NAME:
+ return "Invalid docker container name";
+ case INVALID_DOCKER_IMAGE_NAME:
+ return "Invalid docker image name";
+ case INVALID_DOCKER_USER_NAME:
+ return "Invalid docker user name";
+ case INVALID_DOCKER_INSPECT_FORMAT:
+ return "Invalid docker inspect format";
+ case UNKNOWN_DOCKER_COMMAND:
+ return "Unknown docker command";
+ case INVALID_DOCKER_NETWORK:
+ return "Invalid docker network";
+ case INVALID_DOCKER_CAPABILITY:
+ return "Invalid docker capability";
+ case PRIVILEGED_CONTAINERS_DISABLED:
+ return "Privileged containers are disabled";
+ case INVALID_DOCKER_MOUNT:
+ return "Invalid docker mount";
+ case INVALID_DOCKER_RO_MOUNT:
+ return "Invalid docker read-only mount";
+ case INVALID_DOCKER_RW_MOUNT:
+ return "Invalid docker read-write mount";
+ case MOUNT_ACCESS_ERROR:
+ return "Mount access error";
+ case INVALID_DOCKER_DEVICE:
+ return "Invalid docker device";
+ default:
+ return "Unknown error";
+ }
+}
+
+char *get_docker_binary(const struct configuration *conf) {
+ char *docker_binary = NULL;
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (docker_binary == NULL) {
+ docker_binary = get_configuration_value(DOCKER_BINARY_KEY, "", conf);
+ if (docker_binary == NULL) {
+ docker_binary = strdup("/usr/bin/docker");
+ }
+ }
+ return docker_binary;
+}
+
+int docker_module_enabled(const struct configuration *conf) {
+ struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ if (section != NULL) {
+ return module_enabled(section, CONTAINER_EXECUTOR_CFG_DOCKER_SECTION);
+ }
+ return 0;
+}
+
+int get_docker_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ struct configuration command_config = {0, NULL};
+ ret = read_config(command_file, &command_config);
+ if (ret != 0) {
+ return INVALID_COMMAND_FILE;
+ }
+
+ char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
+ return get_docker_inspect_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
+ return get_docker_load_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
+ return get_docker_pull_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
+ return get_docker_rm_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
+ return get_docker_run_command(command_file, conf, out, outlen);
+ } else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
+ return get_docker_stop_command(command_file, conf, out, outlen);
+ } else {
+ return UNKNOWN_DOCKER_COMMAND;
+ }
+}
+
+int get_docker_inspect_command(const char *command_file, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ const char *valid_format_strings[] = { "{{.State.Status}}",
+ "{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}" };
+ int ret = 0, i = 0, valid_format = 0;
+ char *format = NULL, *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (format == NULL) {
+ free(container_name);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+ for (i = 0; i < 2; ++i) {
+ if (strcmp(format, valid_format_strings[i]) == 0) {
+ valid_format = 1;
+ break;
+ }
+ }
+ if (valid_format != 1) {
+ fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
+ free(container_name);
+ free(format);
+ return INVALID_DOCKER_INSPECT_FORMAT;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(container_name);
+ free(format);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_INSPECT_COMMAND);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " --format=");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, format);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ ret = add_to_buffer(out, outlen, container_name);
+ if (ret != 0) {
+ goto free_and_exit;
+ }
+ free(format);
+ free(container_name);
+ return 0;
+
+ free_and_exit:
+ free(format);
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_load_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_LOAD_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --i=", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int validate_docker_image_name(const char *image_name) {
+ const char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+ return execute_regex_match(regex_str, image_name);
+}
+
+int get_docker_pull_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *image_name = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_PULL_COMMAND);
+ if (ret == 0) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " ", image_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer);
+ free(image_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(image_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_rm_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RM_COMMAND);
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+int get_docker_stop_command(const char *command_file, const struct configuration *conf,
+ char *out, const size_t outlen) {
+ int ret = 0;
+ size_t len = 0, i = 0;
+ char *value = NULL;
+ char *container_name = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+
+ memset(out, 0, outlen);
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_STOP_COMMAND);
+ if (ret == 0) {
+ value = get_configuration_value("time", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (value != NULL) {
+ len = strlen(value);
+ for (i = 0; i < len; ++i) {
+ if (isdigit(value[i]) == 0) {
+ fprintf(ERRORFILE, "Value for time is not a number '%s'\n", value);
+ free(container_name);
+ memset(out, 0, outlen);
+ return INVALID_DOCKER_STOP_COMMAND;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " --time=");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, value);
+ }
+ if (ret != 0) {
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ ret = add_to_buffer(out, outlen, " ");
+ if (ret == 0) {
+ ret = add_to_buffer(out, outlen, container_name);
+ }
+ free(container_name);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ return 0;
+ }
+ free(container_name);
+ return BUFFER_TOO_SMALL;
+}
+
+static int detach_container(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "detach", "-d ", 0, out, outlen);
+}
+
+static int rm_container_on_exit(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "rm", "--rm ", 0, out, outlen);
+}
+
+static int set_container_workdir(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "workdir", "--workdir=", 1, out, outlen);
+}
+
+static int set_cgroup_parent(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "cgroup-parent", "--cgroup-parent=", 1, out, outlen);
+}
+
+static int set_hostname(const struct configuration *command_config, char *out, const size_t outlen) {
+ return add_param_to_command(command_config, "hostname", "--hostname=", 1, out, outlen);
+}
+
+static int set_group_add(const struct configuration *command_config, char *out, const size_t outlen) {
+ int i = 0, ret = 0;
+ char **group_add = get_configuration_values_delimiter("group-add", DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ size_t tmp_buffer_size = 4096;
+ char *tmp_buffer = NULL;
+
+ if (group_add != NULL) {
+ for (i = 0; group_add[i] != NULL; ++i) {
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--group-add ", group_add[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ }
+ return ret;
+}
+
+static int set_network(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "net",
+ "docker.allowed.networks", "--net=",
+ 0, 0, out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Could not find requested network in allowed networks\n");
+ ret = INVALID_DOCKER_NETWORK;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_capabilities(const struct configuration *command_config,
+ const struct configuration *conf, char *out,
+ const size_t outlen) {
+
+ int ret = 0;
+
+ ret = add_to_buffer(out, outlen, "--cap-drop='ALL' ");
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ ret = add_param_to_command_if_allowed(command_config, conf, "cap-add",
+ "docker.allowed.capabilities",
+ "--cap-add=", 1, 0,
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker capability requested\n");
+ ret = INVALID_DOCKER_CAPABILITY;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+static int set_devices(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ int ret = 0;
+ ret = add_param_to_command_if_allowed(command_config, conf, "devices", "docker.allowed.devices", "--device=", 1, ':',
+ out, outlen);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Invalid docker device requested\n");
+ ret = INVALID_DOCKER_DEVICE;
+ memset(out, 0, outlen);
+ }
+
+ return ret;
+}
+
+/**
+ * Helper function to help normalize mounts for checking if mounts are
+ * permitted. The function does the following -
+ * 1. Find the canonical path for mount using realpath
+ * 2. If the path is a directory, add a '/' at the end (if not present)
+ * 3. Return a copy of the canonicalised path(to be freed by the caller)
+ * @param mount path to be canonicalised
+ * @return pointer to canonicalised path, NULL on error
+ */
+static char* normalize_mount(const char* mount) {
+ int ret = 0;
+ struct stat buff;
+ char *ret_ptr = NULL, *real_mount = NULL;
+ if (mount == NULL) {
+ return NULL;
+ }
+ real_mount = realpath(mount, NULL);
+ if (real_mount == NULL) {
+ fprintf(ERRORFILE, "Could not determine real path of mount '%s'\n", mount);
+ free(real_mount);
+ return NULL;
+ }
+ ret = stat(real_mount, &buff);
+ if (ret == 0) {
+ if (S_ISDIR(buff.st_mode)) {
+ size_t len = strlen(real_mount);
+ if (len <= 0) {
+ return NULL;
+ }
+ if (real_mount[len - 1] != '/') {
+ ret_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char));
+ strncpy(ret_ptr, real_mount, len);
+ ret_ptr[len] = '/';
+ ret_ptr[len + 1] = '\0';
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ ret_ptr = strdup(real_mount);
+ }
+ } else {
+ fprintf(ERRORFILE, "Could not stat path '%s'\n", real_mount);
+ ret_ptr = NULL;
+ }
+ free(real_mount);
+ return ret_ptr;
+}
+
+static int normalize_mounts(char **mounts) {
+ int i = 0;
+ char *tmp = NULL;
+ if (mounts == NULL) {
+ return 0;
+ }
+ for (i = 0; mounts[i] != NULL; ++i) {
+ tmp = normalize_mount(mounts[i]);
+ if (tmp == NULL) {
+ return -1;
+ }
+ free(mounts[i]);
+ mounts[i] = tmp;
+ }
+ return 0;
+}
+
+static int check_mount_permitted(const char **permitted_mounts, const char *requested) {
+ int i = 0, ret = 0;
+ size_t permitted_mount_len = 0;
+ char *normalized_path = normalize_mount(requested);
+ if (permitted_mounts == NULL) {
+ return 0;
+ }
+ if (normalized_path == NULL) {
+ return -1;
+ }
+ for (i = 0; permitted_mounts[i] != NULL; ++i) {
+ if (strcmp(normalized_path, permitted_mounts[i]) == 0) {
+ ret = 1;
+ break;
+ }
+ // directory check
+ permitted_mount_len = strlen(permitted_mounts[i]);
+ if (permitted_mount_len > 0
+ && permitted_mounts[i][permitted_mount_len - 1] == '/') {
+ if (strncmp(normalized_path, permitted_mounts[i], permitted_mount_len) == 0) {
+ ret = 1;
+ break;
+ }
+ }
+
+ }
+ free(normalized_path);
+ return ret;
+}
+
+static char* get_mount_source(const char *mount) {
+ char *src_mount = NULL;
+ const char *tmp = NULL;
+ tmp = strchr(mount, ':');
+ if (tmp == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s'\n", mount);
+ return NULL;
+ }
+ src_mount = strndup(mount, tmp - mount);
+ return src_mount;
+}
+
+static int add_mounts(const struct configuration *command_config, const struct configuration *conf, const char *key,
+ const int ro, char *out, const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ const char *ro_suffix = "";
+ const char *tmp_path_buffer[2] = {NULL, NULL};
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char **permitted_ro_mounts = get_configuration_values_delimiter("docker.allowed.ro-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
+ CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
+ char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
+ char *tmp_buffer_2 = NULL, *mount_src = NULL;
+ const char *container_executor_cfg_path = normalize_mount(get_config_path(""));
+ int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
+ if (ro != 0) {
+ ro_suffix = ":ro";
+ }
+
+ if (values != NULL) {
+ ret = normalize_mounts(permitted_ro_mounts);
+ ret |= normalize_mounts(permitted_rw_mounts);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Unable to find permitted docker mounts on disk\n");
+ ret = MOUNT_ACCESS_ERROR;
+ goto free_and_exit;
+ }
+ for (i = 0; values[i] != NULL; ++i) {
+ mount_src = get_mount_source(values[i]);
+ if (mount_src == NULL) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ permitted_rw = check_mount_permitted((const char **) permitted_rw_mounts, mount_src);
+ permitted_ro = check_mount_permitted((const char **) permitted_ro_mounts, mount_src);
+ if (permitted_ro == -1 || permitted_rw == -1) {
+ fprintf(ERRORFILE, "Invalid docker mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_MOUNT;
+ goto free_and_exit;
+ }
+ // rw mount
+ if (ro == 0) {
+ if (permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker rw mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ } else {
+ // determine if the user can modify the container-executor.cfg file
+ tmp_path_buffer[0] = normalize_mount(mount_src);
+ // just re-use the function, flip the args to check if the container-executor path is in the requested
+ // mount point
+ ret = check_mount_permitted(tmp_path_buffer, container_executor_cfg_path);
+ free((void *) tmp_path_buffer[0]);
+ if (ret == 1) {
+ fprintf(ERRORFILE, "Attempting to mount a parent directory '%s' of container-executor.cfg as read-write\n",
+ values[i]);
+ ret = INVALID_DOCKER_RW_MOUNT;
+ goto free_and_exit;
+ }
+ }
+ }
+ //ro mount
+ if (ro != 0 && permitted_ro == 0 && permitted_rw == 0) {
+ fprintf(ERRORFILE, "Invalid docker ro mount '%s', realpath=%s\n", values[i], mount_src);
+ ret = INVALID_DOCKER_RO_MOUNT;
+ goto free_and_exit;
+ }
+ tmp_buffer_2 = (char *) alloc_and_clear_memory(strlen(values[i]) + strlen(ro_suffix) + 1, sizeof(char));
+ strncpy(tmp_buffer_2, values[i], strlen(values[i]));
+ strncpy(tmp_buffer_2 + strlen(values[i]), ro_suffix, strlen(ro_suffix));
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "-v ", tmp_buffer_2);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ free(tmp_buffer_2);
+ free(mount_src);
+ tmp_buffer_2 = NULL;
+ mount_src = NULL;
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ goto free_and_exit;
+ }
+ }
+ }
+
+ free_and_exit:
+ free_values(permitted_ro_mounts);
+ free_values(permitted_rw_mounts);
+ free_values(values);
+ free(mount_src);
+ free((void *) container_executor_cfg_path);
+ free(tmp_buffer);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+static int add_ro_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "ro-mounts", 1, out, outlen);
+}
+
+static int add_rw_mounts(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
+}
+
+static int set_privileged(const struct configuration *command_config, const struct configuration *conf, char *out,
+ const size_t outlen) {
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+ char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config);
+ char *privileged_container_enabled
+ = get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
+ int ret = 0;
+
+ if (value != NULL && strcmp(value, "true") == 0) {
+ if (privileged_container_enabled != NULL) {
+ if (strcmp(privileged_container_enabled, "1") == 0) {
+ ret = add_to_buffer(out, outlen, "--privileged ");
+ if (ret != 0) {
+ ret = BUFFER_TOO_SMALL;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ } else {
+ fprintf(ERRORFILE, "Privileged containers are disabled\n");
+ ret = PRIVILEGED_CONTAINERS_DISABLED;
+ goto free_and_exit;
+ }
+ }
+
+ free_and_exit:
+ free(tmp_buffer);
+ free(value);
+ free(privileged_container_enabled);
+ if (ret != 0) {
+ memset(out, 0, outlen);
+ }
+ return ret;
+}
+
+int get_docker_run_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) {
+ int ret = 0, i = 0;
+ char *container_name = NULL, *user = NULL, *image = NULL;
+ size_t tmp_buffer_size = 1024;
+ char *tmp_buffer = NULL;
+ char **launch_command = NULL;
+ struct configuration command_config = {0, NULL};
+ ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
+ if (ret != 0) {
+ return ret;
+ }
+
+ container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (container_name == NULL || validate_container_name(container_name) != 0) {
+ return INVALID_DOCKER_CONTAINER_NAME;
+ }
+ user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (user == NULL) {
+ return INVALID_DOCKER_USER_NAME;
+ }
+ image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
+ if (image == NULL || validate_docker_image_name(image) != 0) {
+ return INVALID_DOCKER_IMAGE_NAME;
+ }
+
+ ret = add_docker_config_param(&command_config, out, outlen);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ ret = add_to_buffer(out, outlen, DOCKER_RUN_COMMAND);
+ if(ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+
+ tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char));
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, " --name=", container_name);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "--user=", user);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+ memset(tmp_buffer, 0, tmp_buffer_size);
+
+ ret = detach_container(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = rm_container_on_exit(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_container_workdir(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_network(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_ro_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = add_rw_mounts(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_cgroup_parent(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_privileged(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_capabilities(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_hostname(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_group_add(&command_config, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ ret = set_devices(&command_config, conf, out, outlen);
+ if (ret != 0) {
+ return ret;
+ }
+
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", image);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ return BUFFER_TOO_SMALL;
+ }
+
+ launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
+ ",");
+ if (launch_command != NULL) {
+ for (i = 0; launch_command[i] != NULL; ++i) {
+ memset(tmp_buffer, 0, tmp_buffer_size);
+ quote_and_append_arg(&tmp_buffer, &tmp_buffer_size, "", launch_command[i]);
+ ret = add_to_buffer(out, outlen, tmp_buffer);
+ if (ret != 0) {
+ free_values(launch_command);
+ free(tmp_buffer);
+ return BUFFER_TOO_SMALL;
+ }
+ }
+ free_values(launch_command);
+ }
+ free(tmp_buffer);
+ return 0;
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
new file mode 100644
index 0000000..37ec880
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_DOCKER_UTIL_H__
+
+#include "configuration.h"
+
+#define CONTAINER_EXECUTOR_CFG_DOCKER_SECTION "docker"
+#define DOCKER_BINARY_KEY "docker.binary"
+#define DOCKER_COMMAND_FILE_SECTION "docker-command-execution"
+#define DOCKER_INSPECT_COMMAND "inspect"
+#define DOCKER_LOAD_COMMAND "load"
+#define DOCKER_PULL_COMMAND "pull"
+#define DOCKER_RM_COMMAND "rm"
+#define DOCKER_RUN_COMMAND "run"
+#define DOCKER_STOP_COMMAND "stop"
+
+
+enum docker_error_codes {
+ INVALID_COMMAND_FILE = 1,
+ INCORRECT_COMMAND,
+ BUFFER_TOO_SMALL,
+ INVALID_DOCKER_CONTAINER_NAME,
+ INVALID_DOCKER_IMAGE_NAME,
+ INVALID_DOCKER_USER_NAME,
+ INVALID_DOCKER_INSPECT_FORMAT,
+ UNKNOWN_DOCKER_COMMAND,
+ INVALID_DOCKER_NETWORK,
+ INVALID_DOCKER_CAPABILITY,
+ PRIVILEGED_CONTAINERS_DISABLED,
+ INVALID_DOCKER_MOUNT,
+ INVALID_DOCKER_RO_MOUNT,
+ INVALID_DOCKER_RW_MOUNT,
+ MOUNT_ACCESS_ERROR,
+ INVALID_DOCKER_DEVICE,
+ INVALID_DOCKER_STOP_COMMAND
+};
+
+/**
+ * Get the full path for the docker binary.
+ * @param conf Configuration for the container-executor
+ * @return String containing the docker binary to be freed by the user.
+ */
+char *get_docker_binary(const struct configuration *conf);
+
+/**
+ * Get the Docker command line string. The function will inspect the params file to determine the command to be run.
+ * @param command_file File containing the params for the Docker command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the Docker command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker inspect command line string. The function will verify that the params file is meant for the
+ * inspect command.
+ * @param command_file File containing the params for the Docker inspect command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the inspect command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_inspect_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker load command line string. The function will verify that the params file is meant for the load command.
+ * @param command_file File containing the params for the Docker load command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the load command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_load_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker pull command line string. The function will verify that the params file is meant for the pull command.
+ * @param command_file File containing the params for the Docker pull command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the pull command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_pull_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker rm command line string. The function will verify that the params file is meant for the rm command.
+ * @param command_file File containing the params for the Docker rm command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the rm command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_rm_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker run command line string. The function will verify that the params file is meant for the run command.
+ * @param command_file File containing the params for the Docker run command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the run command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_run_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Get the Docker stop command line string. The function will verify that the params file is meant for the stop command.
+ * @param command_file File containing the params for the Docker stop command
+ * @param conf Configuration struct containing the container-executor.cfg details
+ * @param out Buffer to fill with the stop command
+ * @param outlen Size of the output buffer
+ * @return Return code with 0 indicating success and non-zero codes indicating error
+ */
+int get_docker_stop_command(const char* command_file, const struct configuration* conf, char *out, const size_t outlen);
+
+/**
+ * Give an error message for the supplied error code
+ * @param error_code the error code
+ * @return const string containing the error message
+ */
+const char *get_docker_error_message(const int error_code);
+
+/**
+ * Determine if the docker module is enabled in the config.
+ * @param conf Configuration structure
+ * @return 1 if enabled, 0 otherwise
+ */
+int docker_module_enabled(const struct configuration *conf);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index d19c084..40b2c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -21,7 +21,6 @@
#include <errno.h>
#include <strings.h>
#include <string.h>
-#include <stdio.h>
#include <stdlib.h>
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
new file mode 100644
index 0000000..339bec9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/docker-container-executor.cfg
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[docker]
+privieged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 64ee717..609dd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -19,6 +19,7 @@
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
+#include "get_executable.h"
#include "test/test-container-executor-common.h"
#include <inttypes.h>
@@ -1155,145 +1156,6 @@ void test_trim_function() {
free(trimmed);
}
-void test_sanitize_docker_command() {
-
- char *input[] = {
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
- "run ''''''''",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
- char *expected_output[] = {
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
- "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
- "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}' container_e111_1111111111111_1111_01_111111",
- "rm container_e111_1111111111111_1111_01_111111",
- "stop container_e111_1111111111111_1111_01_111111",
- "pull ubuntu",
- "pull registry.com/user/ubuntu",
- "--config=/yarn/local/cdir/ pull registry.com/user/ubuntu"
- };
-
- int input_size = sizeof(input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < input_size; i++) {
- char *command = (char *) calloc(strlen(input[i]) + 1 , sizeof(char));
- strncpy(command, input[i], strlen(input[i]));
- char *op = sanitize_docker_command(command);
- if(strncmp(expected_output[i], op, strlen(expected_output[i])) != 0) {
- printf("FAIL: expected output %s does not match actual output '%s'\n", expected_output[i], op);
- exit(1);
- }
- free(command);
- }
-}
-
-void test_validate_docker_image_name() {
-
- char *good_input[] = {
- "ubuntu",
- "ubuntu:latest",
- "ubuntu:14.04",
- "ubuntu:LATEST",
- "registry.com:5000/user/ubuntu",
- "registry.com:5000/user/ubuntu:latest",
- "registry.com:5000/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu",
- "registry.com/user/ubuntu:latest",
- "registry.com/user/ubuntu:0.1.2.3",
- "registry.com/user/ubuntu:test-image",
- "registry.com/user/ubuntu:test_image",
- "registry.com/ubuntu",
- "user/ubuntu",
- "user/ubuntu:0.1.2.3",
- "user/ubuntu:latest",
- "user/ubuntu:test_image",
- "user/ubuntu.test:test_image",
- "user/ubuntu-test:test-image",
- "registry.com/ubuntu/ubuntu/ubuntu"
- };
-
- char *bad_input[] = {
- "UBUNTU",
- "registry.com|5000/user/ubuntu",
- "registry.com | 5000/user/ubuntu",
- "ubuntu' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''",
- "bad_host_name:5000/user/ubuntu",
- "registry.com:foo/ubuntu/ubuntu/ubuntu",
- "registry.com/ubuntu:foo/ubuntu/ubuntu"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_docker_image_name(good_input[i]);
- if(0 != op) {
- printf("\nFAIL: docker image name %s is invalid", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_docker_image_name(bad_input[j]);
- if(1 != op) {
- printf("\nFAIL: docker image name %s is valid, expected invalid", bad_input[j]);
- exit(1);
- }
- }
-}
-
-void test_validate_container_id() {
- char *good_input[] = {
- "container_e134_1499953498516_50875_01_000007",
- "container_1499953498516_50875_01_000007",
- "container_e1_12312_11111_02_000001"
- };
-
- char *bad_input[] = {
- "CONTAINER",
- "container_e1_12312_11111_02_000001 | /tmp/file"
- "container_e1_12312_11111_02_000001 || # /tmp/file",
- "container_e1_12312_11111_02_000001 # /tmp/file",
- "container_e1_12312_11111_02_000001' || touch /tmp/file #",
- "ubuntu || touch /tmp/file #",
- "''''''''"
- };
-
- int good_input_size = sizeof(good_input) / sizeof(char *);
- int i = 0;
- for(i = 0; i < good_input_size; i++) {
- int op = validate_container_id(good_input[i]);
- if(1 != op) {
- printf("FAIL: docker container name %s is invalid\n", good_input[i]);
- exit(1);
- }
- }
-
- int bad_input_size = sizeof(bad_input) / sizeof(char *);
- int j = 0;
- for(j = 0; j < bad_input_size; j++) {
- int op = validate_container_id(bad_input[j]);
- if(0 != op) {
- printf("FAIL: docker container name %s is valid, expected invalid\n", bad_input[j]);
- exit(1);
- }
- }
-}
-
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1388,15 +1250,6 @@ int main(int argc, char **argv) {
printf("\nTesting is_feature_enabled()\n");
test_is_feature_enabled();
- printf("\nTesting sanitize docker commands()\n");
- test_sanitize_docker_command();
-
- printf("\nTesting validate_docker_image_name()\n");
- test_validate_docker_image_name();
-
- printf("\nTesting validate_container_id()\n");
- test_validate_container_id();
-
test_check_user(0);
#ifdef __APPLE__
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
index 2ec7b2a..b96dea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
@@ -17,7 +17,7 @@
*/
#include <gtest/gtest.h>
-#include <sstream>
+#include <vector>
extern "C" {
#include "util.h"
@@ -135,4 +135,39 @@ namespace ContainerExecutor {
ASSERT_STREQ("foo", trimmed);
free(trimmed);
}
+
+ TEST_F(TestUtil, test_escape_single_quote) {
+ std::vector<std::pair<std::string, std::string> > input_output_vec;
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'abcd'", "'\"'\"'abcd'\"'\"'"));
+ input_output_vec.push_back(std::make_pair<std::string, std::string>("'", "'\"'\"'"));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = input_output_vec.begin(); itr != input_output_vec.end(); ++itr) {
+ char *ret = escape_single_quote(itr->first.c_str());
+ ASSERT_STREQ(itr->second.c_str(), ret);
+ free(ret);
+ }
+ }
+
+ TEST_F(TestUtil, test_quote_and_append_arg) {
+
+ char *tmp = static_cast<char *>(malloc(4096));
+ size_t tmp_size = 4096;
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "ab'cd");
+ ASSERT_STREQ("param='ab'\"'\"'cd' ", tmp);
+ free(tmp);
+
+ tmp = static_cast<char *>(malloc(4));
+ tmp_size = 4;
+ memset(tmp, 0, tmp_size);
+ quote_and_append_arg(&tmp, &tmp_size, "param=", "argument1");
+ ASSERT_STREQ("param='argument1' ", tmp);
+ ASSERT_EQ(1040, tmp_size);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
index 037816a..b259c6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -88,6 +88,39 @@
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
-}
+ }
+
+ TEST_F(TestStringUtils, test_validate_container_id) {
+
+ const char *good_input[] = {
+ "container_e134_1499953498516_50875_01_000007",
+ "container_1499953498516_50875_01_000007",
+ "container_e1_12312_11111_02_000001"
+ };
+
+ const char *bad_input[] = {
+ "CONTAINER",
+ "container_e1_12312_11111_02_000001 | /tmp/file"
+ "container_e1_12312_11111_02_000001 || # /tmp/file",
+ "container_e1_12312_11111_02_000001 # /tmp/file",
+ "container_e1_12312_11111_02_000001' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_container_id(good_input[i]);
+ ASSERT_EQ(1, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_container_id(bad_input[j]);
+ ASSERT_EQ(0, op);
+ }
+ }
-} // namespace ContainerExecutor
\ No newline at end of file
+} // namespace ContainerExecutor
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
new file mode 100644
index 0000000..c627ca8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
@@ -0,0 +1,1122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <fstream>
+#include "errno.h"
+
+extern "C" {
+#include "utils/docker-util.c"
+}
+
+namespace ContainerExecutor {
+
+ class TestDockerUtil : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ docker_command_file = "docker-command.cmd";
+ container_executor_cfg_file = "container-executor.cfg";
+ container_executor_cfg.size = 0;
+ container_executor_cfg.sections = NULL;
+ }
+
+ virtual void TearDown() {
+ remove(docker_command_file.c_str());
+ remove(container_executor_cfg_file.c_str());
+ delete_ce_file();
+ }
+
+ struct configuration container_executor_cfg;
+ std::string docker_command_file;
+ std::string container_executor_cfg_file;
+
+
+ void write_file(const std::string fname, const std::string contents) {
+ std::ofstream command_file;
+ command_file.open(fname.c_str());
+ command_file << contents;
+ command_file.close();
+ }
+
+ int create_ce_file() {
+ int ret = 0;
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ ret = mkdir("../etc", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ ret = mkdir("../etc/hadoop", 0755);
+ if (ret == 0 || errno == EEXIST) {
+ write_file("../etc/hadoop/container-executor.cfg", "");
+ return 0;
+ } else {
+ std::cerr << "Could not create ../etc/hadoop, " << strerror(errno) << std::endl;
+ }
+ } else {
+ std::cerr << "Could not create ../etc, " << strerror(errno) << std::endl;
+ }
+ }
+ std::cerr << "Could not create " << fname << std::endl;
+ return 1;
+ }
+
+ void delete_ce_file() {
+ const char *fname = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ if (strcmp("../etc/hadoop/container-executor.cfg", fname) == 0) {
+ struct stat buffer;
+ if (stat(fname, &buffer) == 0) {
+ remove("../etc/hadoop/container-executor.cfg");
+ rmdir("../etc/hadoop");
+ rmdir("../etc");
+ }
+ }
+ }
+
+ void write_container_executor_cfg(const std::string contents) {
+ write_file(container_executor_cfg_file, contents);
+ }
+
+ void write_command_file(const std::string contents) {
+ write_file(docker_command_file, contents);
+ }
+
+ void run_docker_command_test(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ const std::vector<std::pair<std::string, int> > &bad_file_cmd_vec,
+ int (*docker_func)(const char *, const struct configuration *, char *, const size_t)) {
+ char tmp[8192];
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret) << " for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), tmp);
+ }
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+ for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
+ memset(tmp, 0, 8192);
+ write_command_file(itr2->first);
+ int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(itr2->second, ret) << " for " << itr2->first << std::endl;
+ ASSERT_EQ(0, strlen(tmp));
+ }
+ int ret = (*docker_func)("unknown-file", &container_executor_cfg, tmp, 8192);
+ ASSERT_EQ(static_cast<int>(INVALID_COMMAND_FILE), ret);
+ }
+
+ void run_docker_run_helper_function(const std::vector<std::pair<std::string, std::string> > &file_cmd_vec,
+ int (*helper_func)(const struct configuration *, char *, const size_t)) {
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ for(itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ int ret = read_config(docker_command_file.c_str(), &cfg);
+ if(ret == 0) {
+ ret = (*helper_func)(&cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ }
+ }
+ };
+
+ TEST_F(TestDockerUtil, test_docker_inspect) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n"
+ " format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}}\n"
+ " name=container_e1_12312_11111_02_000001",
+ "inspect --format={{range(.NetworkSettings.Networks)}}{{.IPAddress}},{{end}}{{.Config.Hostname}} container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(
+ std::make_pair<std::string, int>("docker-command=inspect\n format='{{.State.Status}}'",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}\n name=",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.State.Status}}",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format=\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=inspect\n format={{.IPAddress}}\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INVALID_DOCKER_INSPECT_FORMAT)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_inspect_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_load) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n image=image-id",
+ "load --i='image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=load\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=load\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=load",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_load_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_validate_image_name) {
+ const char *good_input[] = {
+ "ubuntu",
+ "ubuntu:latest",
+ "ubuntu:14.04",
+ "ubuntu:LATEST",
+ "registry.com:5000/user/ubuntu",
+ "registry.com:5000/user/ubuntu:latest",
+ "registry.com:5000/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu",
+ "registry.com/user/ubuntu:latest",
+ "registry.com/user/ubuntu:0.1.2.3",
+ "registry.com/user/ubuntu:test-image",
+ "registry.com/user/ubuntu:test_image",
+ "registry.com/ubuntu",
+ "user/ubuntu",
+ "user/ubuntu:0.1.2.3",
+ "user/ubuntu:latest",
+ "user/ubuntu:test_image",
+ "user/ubuntu.test:test_image",
+ "user/ubuntu-test:test-image",
+ "registry.com/ubuntu/ubuntu/ubuntu"
+ };
+
+ const char *bad_input[] = {
+ "UBUNTU",
+ "registry.com|5000/user/ubuntu",
+ "registry.com | 5000/user/ubuntu",
+ "ubuntu' || touch /tmp/file #",
+ "ubuntu || touch /tmp/file #",
+ "''''''''",
+ "bad_host_name:5000/user/ubuntu",
+ "registry.com:foo/ubuntu/ubuntu/ubuntu",
+ "registry.com/ubuntu:foo/ubuntu/ubuntu"
+ };
+
+ int good_input_size = sizeof(good_input) / sizeof(char *);
+ int i = 0;
+ for (i = 0; i < good_input_size; i++) {
+ int op = validate_docker_image_name(good_input[i]);
+ ASSERT_EQ(0, op);
+ }
+
+ int bad_input_size = sizeof(bad_input) / sizeof(char *);
+ int j = 0;
+ for (j = 0; j < bad_input_size; j++) {
+ int op = validate_docker_image_name(bad_input[j]);
+ ASSERT_EQ(1, op);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_pull) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n image=image-id",
+ "pull 'image-id' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=pull\n image=image-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=pull\n image=", static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>("[docker-command-execution]\n docker-command=pull",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_pull_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_rm) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(
+ std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n name=container_e1_12312_11111_02_000001",
+ "rm container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=rm\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=rm", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_rm_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_stop) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001",
+ "stop container_e1_12312_11111_02_000001"));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\ntime=25",
+ "stop --time=25 container_e1_12312_11111_02_000001"));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001",
+ static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "docker-command=stop\n name=ctr-id", static_cast<int>(INCORRECT_COMMAND)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop", static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=stop\n name=container_e1_12312_11111_02_000001\n time=abcd",
+ static_cast<int>(INVALID_DOCKER_STOP_COMMAND)));
+
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_stop_command);
+ }
+
+ TEST_F(TestDockerUtil, test_detach_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n detach=true", "-d "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, detach_container);
+ }
+
+ TEST_F(TestDockerUtil, test_rm_container) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rm=true", "--rm "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, rm_container_on_exit);
+ }
+
+ TEST_F(TestDockerUtil, test_set_container_workdir) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n workdir=/tmp/test", "--workdir='/tmp/test' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_container_workdir);
+ }
+
+ TEST_F(TestDockerUtil, test_set_cgroup_parent) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cgroup-parent=/sys/fs/cgroup/yarn",
+ "--cgroup-parent='/sys/fs/cgroup/yarn' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_cgroup_parent);
+ }
+
+ TEST_F(TestDockerUtil, test_set_hostname) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n hostname=ctr-id", "--hostname='ctr-id' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_hostname);
+ }
+
+ TEST_F(TestDockerUtil, test_set_group_add) {
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n group-add=1000,1001", "--group-add '1000' --group-add '1001' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+
+ run_docker_run_helper_function(file_cmd_vec, set_group_add);
+ }
+
+ TEST_F(TestDockerUtil, test_set_network) {
+ struct configuration container_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.networks=sdn1,bridge";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=bridge", "--net='bridge' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n net=sdn1", "--net='sdn1' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ struct configuration cmd_cfg;
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_network(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ struct configuration cmd_cfg_1;
+ write_command_file("[docker-command-execution]\n docker-command=run\n net=sdn2");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg_1);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_network(&cmd_cfg_1, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_check_mount_permitted) {
+ const char *permitted_mounts[] = {"/usr/", "/bin/ls", NULL};
+ std::vector<std::pair<std::string, int> > test_data;
+ test_data.push_back(std::make_pair<std::string, int>("/usr", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/usr/", 1));
+ test_data.push_back(std::make_pair<std::string, int>("/bin/ls", 1));
+ test_data.push_back(std::make_pair<std::string, int>("//bin/", 0));
+ test_data.push_back(std::make_pair<std::string, int>("/tmp/random-file", -1));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr;
+ for (itr = test_data.begin(); itr != test_data.end(); ++itr) {
+ int ret = check_mount_permitted(permitted_mounts, itr->first.c_str());
+ ASSERT_EQ(itr->second, ret) << "for input " << itr->first;
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_normalize_mounts) {
+ const int entries = 4;
+ const char *permitted_mounts[] = {"/home", "/usr", "/bin/ls", NULL};
+ const char *expected[] = {"/home/", "/usr/", "/bin/ls", NULL};
+ char **ptr = static_cast<char **>(malloc(entries * sizeof(char *)));
+ for (int i = 0; i < entries; ++i) {
+ if (permitted_mounts[i] != NULL) {
+ ptr[i] = strdup(permitted_mounts[i]);
+ } else {
+ ptr[i] = NULL;
+ }
+ }
+ normalize_mounts(ptr);
+ for (int i = 0; i < entries; ++i) {
+ ASSERT_STREQ(expected[i], ptr[i]);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_privileged) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1",
+ "[docker]\n docker.privileged-containers.enabled=0",
+ "[docker]\n"};
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ write_container_executor_cfg(container_executor_cfg_contents[0]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ // check default case and when it's turned off
+ for (int i = 1; i < 3; ++i) {
+ write_container_executor_cfg(container_executor_cfg_contents[i]);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ file_cmd_vec.clear();
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n privileged=false", ""));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", ""));
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n privileged=true");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_privileged(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_set_capabilities) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.capabilities=CHROOT,MKNOD";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT,MKNOD",
+ "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run", "--cap-drop='ALL' "));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n cap-add=SETGID");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_capabilities(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_set_devices) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.devices=/dev/test-device,/dev/device2";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/test-device:/dev/test-device",
+ "--device='/dev/test-device:/dev/test-device' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n devices=/dev/device2:/dev/device2",
+ "--device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n "
+ "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2",
+ "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ if (ret != 0) {
+ FAIL();
+ }
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device3:/dev/device3");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device1");
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = set_devices(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+
+ TEST_F(TestDockerUtil, test_add_rw_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls,..\n "
+ "docker.allowed.ro-mounts=/bin/cat";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr:/usr", "-v '/usr:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/:/usr", "-v '/usr/:/usr' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin:/mydisk1,/var/log/:/mydisk2",
+ "-v '/usr/bin:/mydisk1' -v '/var/log/:/mydisk2' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/bin/cat:/bin/cat",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ // verify that you can't mount any directory in the container-executor.cfg path
+ char *ce_path = realpath("../etc/hadoop/container-executor.cfg", NULL);
+ while (strlen(ce_path) != 0) {
+ std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n rw-mounts=";
+ cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
+ memset(buff, 0, buff_len);
+ write_command_file(cmd_file_contents);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
+ ASSERT_STREQ("", buff);
+ char *tmp = strrchr(ce_path, '/');
+ if (tmp != NULL) {
+ *tmp = '\0';
+ }
+ }
+ free(ce_path);
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_rw_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_add_ro_mounts) {
+ struct configuration container_cfg, cmd_cfg;
+ const int buff_len = 1024;
+ char buff[buff_len];
+ int ret = 0;
+ std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/usr,/var,/bin/ls\n "
+ "docker.allowed.ro-mounts=/bin/cat,/bin/ln";
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr:/usr", "-v '/usr:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/:/usr", "-v '/usr/:/usr:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ls:/bin/ls", "-v '/bin/ls:/bin/ls:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/ln:/bin/ln", "-v '/bin/ln:/bin/ln:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/bin/cat:/bin/cat",
+ "-v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin:/mydisk1,/bin/cat:/bin/cat",
+ "-v '/usr/bin:/mydisk1:ro' -v '/bin/cat:/bin/cat:ro' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n", ""));
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+
+ for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(0, ret);
+ ASSERT_STREQ(itr->second.c_str(), buff);
+ }
+
+ std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+ bad_file_cmds_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah",
+ static_cast<int>(INVALID_DOCKER_MOUNT)));
+
+ std::vector<std::pair<std::string, int> >::const_iterator itr2;
+
+ for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
+ memset(buff, 0, buff_len);
+ write_command_file(itr2->first);
+ ret = read_config(docker_command_file.c_str(), &cmd_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(itr2->second, ret);
+ ASSERT_STREQ("", buff);
+ }
+
+ container_executor_cfg_contents = "[docker]\n";
+ write_container_executor_cfg(container_executor_cfg_contents);
+ ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ write_command_file("[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home");
+ strcpy(buff, "test string");
+ ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len);
+ ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
+ ASSERT_EQ(0, strlen(buff));
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_privileged) {
+
+ std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test";
+ write_file(container_executor_cfg_file, container_executor_contents);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' "
+ "'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' "
+ "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' "
+ "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test",
+ static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n",
+ static_cast<int>(INVALID_DOCKER_IMAGE_NAME)));
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n",
+ static_cast<int>(INVALID_DOCKER_USER_NAME)));
+
+ // invalid rw mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/var/log:/var/log\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RW_MOUNT)));
+
+ // invalid ro mount
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin:/bin,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_RO_MOUNT)));
+
+ // invalid capability
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_CAPABILITY)));
+
+ // invalid device
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_DEVICE)));
+
+ // invalid network
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(INVALID_DOCKER_NETWORK)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+
+ TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
+
+ std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " docker.allowed.devices=/dev/test",
+ "[docker]\n docker.allowed.ro-mounts=/var,/etc,/bin/ls\n"
+ " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n "
+ " docker.allowed.capabilities=CHOWN,SETUID\n"
+ " privileged=0\n"
+ " docker.allowed.devices=/dev/test"};
+ for (int i = 0; i < 2; ++i) {
+ write_file(container_executor_cfg_file, container_executor_contents[i]);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = create_ce_file();
+ if (ret != 0) {
+ std::cerr << "Could not create ce file, skipping test" << std::endl;
+ return;
+ }
+
+ std::vector<std::pair<std::string, std::string> > file_cmd_vec;
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n"
+ " user=test\n launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'"
+ " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' "
+ "'test_script.sh' 'arg1' 'arg2' "));
+
+ file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'"
+ " -v '/bin/ls:/bin/ls:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' "
+ "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'"
+ " 'test_script.sh' 'arg1' 'arg2' "));
+
+ std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
+ bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
+ "[docker-command-execution]\n"
+ " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n"
+ " ro-mounts=/var/log:/var/log,/var/lib:/lib,/bin/ls:/bin/ls\n rw-mounts=/tmp:/tmp\n"
+ " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n"
+ " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n"
+ " launch-command=bash,test_script.sh,arg1,arg2",
+ static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
+
+ run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_config_param) {
+ std::vector<std::pair<std::string, std::string> > input_output_map;
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=inspect\n docker-config=/my-config\n"
+ " format={{.State.Status}}\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' inspect --format={{.State.Status}} container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=load\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' load --i='image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=pull\n docker-config=/my-config\n image=image-id",
+ "--config='/my-config' pull 'image-id' "));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=rm\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' rm container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=stop\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001",
+ "--config='/my-config' stop container_e1_12312_11111_02_000001"));
+ input_output_map.push_back(std::make_pair<std::string, std::string>(
+ "[docker-command-execution]\n docker-command=run\n docker-config=/my-config\n name=container_e1_12312_11111_02_000001\n"
+ " image=docker-image\n user=test",
+ "--config='/my-config' run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' "));
+
+ std::vector<std::pair<std::string, std::string> >::const_iterator itr;
+ char buffer[4096];
+ struct configuration cfg = {0, NULL};
+ for (itr = input_output_map.begin(); itr != input_output_map.end(); ++itr) {
+ memset(buffer, 0, 4096);
+ write_command_file(itr->first);
+ int ret = get_docker_command(docker_command_file.c_str(), &cfg, buffer, 4096);
+ ASSERT_EQ(0, ret) << "for input " << itr->first;
+ ASSERT_STREQ(itr->second.c_str(), buffer);
+ }
+ }
+
+ TEST_F(TestDockerUtil, test_docker_module_enabled) {
+
+ std::vector<std::pair<std::string, int> > input_out_vec;
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=true", 1));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=false", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n module.enabled=1", 0));
+ input_out_vec.push_back(std::make_pair<std::string, int>("[docker]\n", 0));
+
+ for (size_t i = 0; i < input_out_vec.size(); ++i) {
+ write_file(container_executor_cfg_file, input_out_vec[i].first);
+ int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg);
+ if (ret != 0) {
+ FAIL();
+ }
+ ret = docker_module_enabled(&container_executor_cfg);
+ ASSERT_EQ(input_out_vec[i].second, ret) << " incorrect output for "
+ << input_out_vec[i].first;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index e548790..fbfee54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -295,32 +295,37 @@ public class TestDockerContainerRuntime {
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -347,10 +352,13 @@ public class TestDockerContainerRuntime {
String uid = "";
String gid = "";
+ String[] groups = {};
Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
new String[]{"id", "-u", runAsUser});
Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
new String[]{"id", "-g", runAsUser});
+ Shell.ShellCommandExecutor shexec3 = new Shell.ShellCommandExecutor(
+ new String[]{"id", "-G", runAsUser});
try {
shexec1.execute();
// get rid of newline at the end
@@ -365,37 +373,48 @@ public class TestDockerContainerRuntime {
} catch (Exception e) {
LOG.info("Could not run id -g command: " + e);
}
+ try {
+ shexec3.execute();
+ groups = shexec3.getOutput().replace("\n", " ").split(" ");
+ } catch (Exception e) {
+ LOG.info("Could not run id -G command: " + e);
+ }
uidGidPair = uid + ":" + gid;
- //This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
- .append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=host ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ")
- .append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ")
- .append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ")
- .append("(--group-add \\d+ )*")
- .append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, uidGidPair,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- //Assert.assertEquals(expectedCommand, dockerCommands.get(0));
- Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand));
+ Assert.assertEquals(14, dockerCommands.size());
+ int counter = 0;
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" group-add=" + String.join(",", groups),
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -481,29 +500,38 @@ public class TestDockerContainerRuntime {
String dockerCommandFile = args.get(11);
//This is the expected docker invocation for this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + allowedNetwork + " ")
- .append("--hostname=" + expectedHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
-
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=test.hostname",
+ dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" net=" + allowedNetwork, dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -537,30 +565,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork1
// ("sdn1") is the expected network to be used in this case
- StringBuffer expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork1 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- String expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
-
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
-
+ int expected = 13;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn1", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
//now set an explicit (non-default) allowedNetwork and ensure that it is
// used.
@@ -575,28 +610,37 @@ public class TestDockerContainerRuntime {
//This is the expected docker invocation for this case. customNetwork2
// ("sdn2") is the expected network to be used in this case
- expectedCommandTemplate =
- new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
- .append("--workdir=%3$s ")
- .append("--net=" + customNetwork2 + " ")
- .append("--hostname=" + defaultHostname + " ")
- .append(getExpectedTestCapabilitiesArgumentString())
- .append(getExpectedCGroupsMountString())
- .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
- .append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
- .append("-v %8$s:%8$s ").append("%9$s ")
- .append("bash %10$s/launch_container.sh");
-
- expectedCommand = String
- .format(expectedCommandTemplate.toString(), containerId, runAsUser,
- containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
- containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
- image, containerWorkDir);
dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+ counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=sdn2", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals(expectedCommand, dockerCommands.get(0));
//disallowed network should trigger a launch failure
@@ -630,7 +674,8 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
+ int expected = 13;
+ Assert.assertEquals(expected, dockerCommands.size());
String command = dockerCommands.get(0);
@@ -738,13 +783,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- //submitting user is whitelisted. ensure --privileged is in the invocation
- Assert.assertTrue("Did not find expected '--privileged' in docker run args "
- + ": " + command, command.contains("--privileged"));
+ int expected = 14;
+ int counter = 0;
+ Assert.assertEquals(expected, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
+ Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++));
+ Assert
+ .assertEquals(" image=busybox:latest", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(counter++));
+ Assert.assertEquals(" net=host", dockerCommands.get(counter++));
+ Assert.assertEquals(" privileged=true", dockerCommands.get(counter++));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(counter++));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(counter++));
}
@Test
@@ -833,15 +900,33 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
-
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount" +
- ":ro "));
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
}
@Test
@@ -885,20 +970,35 @@ public class TestDockerContainerRuntime {
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
- Assert.assertEquals(1, dockerCommands.size());
-
- String command = dockerCommands.get(0);
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(
+ " ro-mounts=/test_local_dir/test_resource_file:test_mount1,"
+ + "/test_local_dir/test_resource_file:test_mount2",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount1 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount1" +
- ":ro "));
- Assert.assertTrue("Did not find expected " +
- "/test_local_dir/test_resource_file:test_mount2 mount in docker " +
- "run args : " + command,
- command.contains(" -v /test_local_dir/test_resource_file:test_mount2" +
- ":ro "));
}
@Test
@@ -930,8 +1030,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.TERM);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -940,8 +1042,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.KILL);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
@Test
@@ -950,8 +1054,10 @@ public class TestDockerContainerRuntime {
IOException {
List<String> dockerCommands = getDockerCommandsForSignal(
ContainerExecutor.Signal.QUIT);
- Assert.assertEquals(1, dockerCommands.size());
- Assert.assertEquals("stop container_id", dockerCommands.get(0));
+ Assert.assertEquals(3, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" docker-command=stop", dockerCommands.get(1));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(2));
}
private List<String> getDockerCommandsForSignal(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/4] hadoop git commit: YARN-6623. Add support to turn off launching
privileged containers in the container-executor. (Varun Vasudev via wangda)
Posted by wa...@apache.org.
YARN-6623. Add support to turn off launching privileged containers in the container-executor. (Varun Vasudev via wangda)
Change-Id: I76aec68cbfb42d239279d7245a03290d691e87a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3b1c631
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3b1c631
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3b1c631
Branch: refs/heads/trunk
Commit: d3b1c6319546706c41a2011ead6c3fe208883200
Parents: c114da5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Sep 28 16:41:09 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Sep 28 16:41:09 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn/conf/container-executor.cfg | 12 +
.../src/CMakeLists.txt | 5 +-
.../runtime/DockerLinuxContainerRuntime.java | 11 +-
.../linux/runtime/docker/DockerClient.java | 25 +-
.../linux/runtime/docker/DockerCommand.java | 55 +-
.../runtime/docker/DockerCommandExecutor.java | 3 +-
.../runtime/docker/DockerInspectCommand.java | 13 +-
.../linux/runtime/docker/DockerLoadCommand.java | 2 +-
.../linux/runtime/docker/DockerPullCommand.java | 2 +-
.../linux/runtime/docker/DockerRmCommand.java | 4 +-
.../linux/runtime/docker/DockerRunCommand.java | 68 +-
.../linux/runtime/docker/DockerStopCommand.java | 6 +-
.../container-executor/impl/configuration.c | 17 +
.../container-executor/impl/configuration.h | 19 +-
.../impl/container-executor.c | 316 +----
.../impl/container-executor.h | 9 -
.../container-executor/impl/get_executable.c | 3 -
.../container-executor/impl/get_executable.h | 29 +
.../main/native/container-executor/impl/main.c | 31 +-
.../impl/modules/common/module-configs.c | 3 +-
.../impl/modules/common/module-configs.h | 1 +
.../main/native/container-executor/impl/util.c | 60 +-
.../main/native/container-executor/impl/util.h | 46 +-
.../container-executor/impl/utils/docker-util.c | 998 ++++++++++++++++
.../container-executor/impl/utils/docker-util.h | 147 +++
.../impl/utils/string-utils.c | 1 -
.../docker-container-executor.cfg | 13 +
.../test/test-container-executor.c | 149 +--
.../native/container-executor/test/test_util.cc | 37 +-
.../test/utils/test-string-utils.cc | 37 +-
.../test/utils/test_docker_util.cc | 1122 ++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 398 ++++---
.../docker/TestDockerCommandExecutor.java | 35 +-
.../docker/TestDockerInspectCommand.java | 29 +-
.../runtime/docker/TestDockerLoadCommand.java | 9 +-
.../runtime/docker/TestDockerPullCommand.java | 8 +-
.../runtime/docker/TestDockerRmCommand.java | 8 +-
.../runtime/docker/TestDockerRunCommand.java | 25 +-
.../runtime/docker/TestDockerStopCommand.java | 15 +-
.../src/site/markdown/DockerContainers.md | 33 +-
40 files changed, 3067 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d68cee8..023654b 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -2,3 +2,15 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
min.user.id=1000#Prevent other super-users
allowed.system.users=##comma separated list of system users who CAN run applications
+feature.tc.enabled=0
+
+# The configs below deal with settings for Docker
+#[docker]
+# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default
+# docker.binary=/usr/bin/docker
+# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
+# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container
+# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
+# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
+# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
+# docker.privileged-containers.enabled=0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 3d5b506..9d83bf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -132,6 +132,7 @@ add_library(container
main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
main/native/container-executor/impl/modules/common/module-configs.c
main/native/container-executor/impl/modules/gpu/gpu-module.c
+ main/native/container-executor/impl/utils/docker-util.c
)
add_executable(container-executor
@@ -156,6 +157,7 @@ output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
add_executable(cetest
+ main/native/container-executor/impl/get_executable.c
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
@@ -163,6 +165,7 @@ add_executable(cetest
main/native/container-executor/test/utils/test-path-utils.cc
main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
main/native/container-executor/test/modules/gpu/test-gpu-module.cc
- main/native/container-executor/test/test_util.cc)
+ main/native/container-executor/test/test_util.cc
+ main/native/container-executor/test/utils/test_docker_util.cc)
target_link_libraries(cetest gtest container)
output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index b8d9b0a..29c6fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -559,8 +559,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
runCommand.setCapabilities(capabilities);
if(cgroupsRootDirectory != null) {
- runCommand.addMountLocation(cgroupsRootDirectory,
- cgroupsRootDirectory + ":ro", false);
+ runCommand.addReadOnlyMountLocation(cgroupsRootDirectory,
+ cgroupsRootDirectory, false);
}
List<String> allDirs = new ArrayList<>(containerLocalDirs);
@@ -584,7 +584,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String src = validateMount(dir[0], localizedResources);
String dst = dir[1];
- runCommand.addMountLocation(src, dst + ":ro", true);
+ runCommand.addReadOnlyMountLocation(src, dst, true);
}
}
}
@@ -626,7 +626,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp, null, null, false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
- LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
+ LOG.info("Docker command used: " + runCommand);
throw new ContainerExecutionException("Launch container failed", e
.getExitCode(), e.getOutput(), e.getErrorOutput());
@@ -757,8 +757,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
launchOp.appendArgs(tcCommandFile);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Launching container with cmd: " + runCommand
- .getCommandWithArguments());
+ LOG.debug("Launching container with cmd: " + runCommand);
}
return launchOp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index 536a22d..77c53a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -23,7 +23,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +34,8 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.util.List;
+import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -68,10 +70,25 @@ public final class DockerClient {
TMP_FILE_SUFFIX, new
File(tmpDirPath));
- Writer writer = new OutputStreamWriter(new FileOutputStream(dockerCommandFile),
- "UTF-8");
+ Writer writer = new OutputStreamWriter(
+ new FileOutputStream(dockerCommandFile), "UTF-8");
PrintWriter printWriter = new PrintWriter(writer);
- printWriter.print(cmd.getCommandWithArguments());
+ printWriter.println("[docker-command-execution]");
+ for (Map.Entry<String, List<String>> entry :
+ cmd.getDockerCommandWithArguments().entrySet()) {
+ if (entry.getKey().contains("=")) {
+ throw new ContainerExecutionException(
+ "'=' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ if (entry.getValue().contains("\n")) {
+ throw new ContainerExecutionException(
+ "'\\n' found in entry for docker command file, key = " + entry
+ .getKey() + "; value = " + entry.getValue());
+ }
+ printWriter.println(" " + entry.getKey() + "=" + StringUtils
+ .join(",", entry.getValue()));
+ }
printWriter.close();
return dockerCommandFile.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
index 3b76a5c..7802209 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommand.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,32 +37,55 @@ import java.util.List;
* e.g 'run', 'load', 'inspect' etc.,
*/
-public abstract class DockerCommand {
+public abstract class DockerCommand {
private final String command;
- private final List<String> commandWithArguments;
+ private final Map<String, List<String>> commandArguments;
protected DockerCommand(String command) {
+ String dockerCommandKey = "docker-command";
this.command = command;
- this.commandWithArguments = new ArrayList<>();
- commandWithArguments.add(command);
+ this.commandArguments = new TreeMap<>();
+ commandArguments.put(dockerCommandKey, new ArrayList<>());
+ commandArguments.get(dockerCommandKey).add(command);
}
- /** Returns the docker sub-command string being used
- * e.g 'run'
+ /**
+ * Returns the docker sub-command string being used
+ * e.g 'run'.
*/
public final String getCommandOption() {
return this.command;
}
- /** Add command commandWithArguments - this method is only meant for use by
- * sub-classes
- * @param arguments to be added
+ /**
+ * Add command commandWithArguments - this method is only meant for use by
+ * sub-classes.
+ *
+ * @param key name of the key to be added
+ * @param value value of the key
*/
- protected final void addCommandArguments(String... arguments) {
- this.commandWithArguments.addAll(Arrays.asList(arguments));
+ protected final void addCommandArguments(String key, String value) {
+ List<String> list = commandArguments.get(key);
+ if (list != null) {
+ list.add(value);
+ return;
+ }
+ list = new ArrayList<>();
+ list.add(value);
+ this.commandArguments.put(key, list);
}
- public String getCommandWithArguments() {
- return StringUtils.join(" ", commandWithArguments);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return Collections.unmodifiableMap(commandArguments);
}
-}
\ No newline at end of file
+
+ @Override
+ public String toString() {
+ StringBuffer ret = new StringBuffer(this.command);
+ for (Map.Entry<String, List<String>> entry : commandArguments.entrySet()) {
+ ret.append(" ").append(entry.getKey());
+ ret.append("=").append(StringUtils.join(",", entry.getValue()));
+ }
+ return ret.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 5739912..76b53af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -88,8 +88,7 @@ public final class DockerCommandExecutor {
dockerOp.disableFailureLogging();
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Running docker command: "
- + dockerCommand.getCommandWithArguments());
+ LOG.debug("Running docker command: " + dockerCommand);
}
try {
String result = privilegedOperationExecutor
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index 812a35f..d27f74d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -26,16 +26,14 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
*/
public class DockerInspectCommand extends DockerCommand {
private static final String INSPECT_COMMAND = "inspect";
- private String containerName;
public DockerInspectCommand(String containerName) {
super(INSPECT_COMMAND);
- this.containerName = containerName;
+ super.addCommandArguments("name", containerName);
}
public DockerInspectCommand getContainerStatus() {
- super.addCommandArguments("--format='{{.State.Status}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{.State.Status}}");
return this;
}
@@ -43,9 +41,8 @@ public class DockerInspectCommand extends DockerCommand {
// Be sure to not use space in the argument, otherwise the
// extract_values_delim method in container-executor binary
// cannot parse the arguments correctly.
- super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}"
- + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'");
- super.addCommandArguments(containerName);
+ super.addCommandArguments("format", "{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}");
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
index e4d92e0..fa2988c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerLoadCommand.java
@@ -25,6 +25,6 @@ public class DockerLoadCommand extends DockerCommand {
public DockerLoadCommand(String localImageFile) {
super(LOAD_COMMAND);
- super.addCommandArguments("--i=" + localImageFile);
+ super.addCommandArguments("image", localImageFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
index 351e09e..5e6108e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java
@@ -25,7 +25,7 @@ public class DockerPullCommand extends DockerCommand {
public DockerPullCommand(String imageName) {
super(PULL_COMMAND);
- super.addCommandArguments(imageName);
+ super.addCommandArguments("image", imageName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index b1aea61..dcfe777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -25,6 +25,6 @@ public class DockerRmCommand extends DockerCommand {
public DockerRmCommand(String containerName) {
super(RM_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 1e1e6e8..c7bf827 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -20,42 +20,39 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.hadoop.util.StringUtils;
-
import java.io.File;
-import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class DockerRunCommand extends DockerCommand {
private static final String RUN_COMMAND = "run";
- private final String image;
- private List<String> overrrideCommandWithArgs;
/** The following are mandatory: */
public DockerRunCommand(String containerId, String user, String image) {
super(RUN_COMMAND);
- super.addCommandArguments("--name=" + containerId, "--user=" + user);
- this.image = image;
+ super.addCommandArguments("name", containerId);
+ super.addCommandArguments("user", user);
+ super.addCommandArguments("image", image);
}
public DockerRunCommand removeContainerOnExit() {
- super.addCommandArguments("--rm");
+ super.addCommandArguments("rm", "true");
return this;
}
public DockerRunCommand detachOnRun() {
- super.addCommandArguments("-d");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand setContainerWorkDir(String workdir) {
- super.addCommandArguments("--workdir=" + workdir);
+ super.addCommandArguments("workdir", workdir);
return this;
}
public DockerRunCommand setNetworkType(String type) {
- super.addCommandArguments("--net=" + type);
+ super.addCommandArguments("net", type);
return this;
}
@@ -65,79 +62,80 @@ public class DockerRunCommand extends DockerCommand {
if (!sourceExists && !createSource) {
return this;
}
- super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+ super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
+ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
+ destinationPath, boolean createSource) {
+ boolean sourceExists = new File(sourcePath).exists();
+ if (!sourceExists && !createSource) {
+ return this;
+ }
+ super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand setCGroupParent(String parentPath) {
- super.addCommandArguments("--cgroup-parent=" + parentPath);
+ super.addCommandArguments("cgroup-parent", parentPath);
return this;
}
/* Run a privileged container. Use with extreme care */
public DockerRunCommand setPrivileged() {
- super.addCommandArguments("--privileged");
+ super.addCommandArguments("privileged", "true");
return this;
}
public DockerRunCommand setCapabilities(Set<String> capabilties) {
//first, drop all capabilities
- super.addCommandArguments("--cap-drop=ALL");
+ super.addCommandArguments("cap-drop", "ALL");
//now, add the capabilities supplied
for (String capability : capabilties) {
- super.addCommandArguments("--cap-add=" + capability);
+ super.addCommandArguments("cap-add", capability);
}
return this;
}
public DockerRunCommand setHostname(String hostname) {
- super.addCommandArguments("--hostname=" + hostname);
+ super.addCommandArguments("hostname", hostname);
return this;
}
public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) {
- super.addCommandArguments("--device=" + sourceDevice + ":" +
+ super.addCommandArguments("devices", sourceDevice + ":" +
destinationDevice);
return this;
}
public DockerRunCommand enableDetach() {
- super.addCommandArguments("--detach=true");
+ super.addCommandArguments("detach", "true");
return this;
}
public DockerRunCommand disableDetach() {
- super.addCommandArguments("--detach=false");
+ super.addCommandArguments("detach", "false");
return this;
}
public DockerRunCommand groupAdd(String[] groups) {
- for(int i = 0; i < groups.length; i++) {
- super.addCommandArguments("--group-add " + groups[i]);
- }
+ super.addCommandArguments("group-add", String.join(",", groups));
return this;
}
public DockerRunCommand setOverrideCommandWithArgs(
List<String> overrideCommandWithArgs) {
- this.overrrideCommandWithArgs = overrideCommandWithArgs;
+ for(String override: overrideCommandWithArgs) {
+ super.addCommandArguments("launch-command", override);
+ }
return this;
}
@Override
- public String getCommandWithArguments() {
- List<String> argList = new ArrayList<>();
-
- argList.add(super.getCommandWithArguments());
- argList.add(image);
-
- if (overrrideCommandWithArgs != null) {
- argList.addAll(overrrideCommandWithArgs);
- }
-
- return StringUtils.join(" ", argList);
+ public Map<String, List<String>> getDockerCommandWithArguments() {
+ return super.getDockerCommandWithArguments();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
index e9d6c43..ccfff82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerStopCommand.java
@@ -29,11 +29,11 @@ public class DockerStopCommand extends DockerCommand {
public DockerStopCommand(String containerName) {
super(STOP_COMMAND);
- super.addCommandArguments(containerName);
+ super.addCommandArguments("name", containerName);
}
public DockerStopCommand setGracePeriod(int value) {
- super.addCommandArguments("--time=" + Integer.toString(value));
+ super.addCommandArguments("time", Integer.toString(value));
return this;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 12dbc4c..f23cff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -21,6 +21,7 @@
#include "configuration.h"
#include "util.h"
+#include "get_executable.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
@@ -696,3 +697,19 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
return 0;
}
+
+char *get_config_path(const char *argv0) {
+ char *executable_file = get_executable((char *) argv0);
+ if (!executable_file) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
+ return NULL;
+ }
+
+ const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
+ char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ if (conf_file == NULL) {
+ fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
+ }
+ return conf_file;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 1ea5561..7fa684e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -23,10 +23,21 @@
#define _WITH_GETLINE
#endif
-#include <stddef.h>
+#include "config.h"
+
+#define CONF_FILENAME "container-executor.cfg"
+
+// When building as part of a Maven build this value gets defined by using
+// container-executor.conf.dir property. See:
+// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+// for details.
+// NOTE: if this ends up being a relative path it gets resolved relative to
+// the location of the container-executor binary itself, not getwd(3)
+#ifndef HADOOP_CONF_DIR
+#error HADOOP_CONF_DIR must be defined
+#endif
-/** Define a platform-independent constant instead of using PATH_MAX */
-#define EXECUTOR_PATH_MAX 4096
+#include <stddef.h>
// Configuration data structures.
struct kv_pair {
@@ -207,4 +218,6 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
+char *get_config_path(const char* argv0);
+
#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index e8bf564..08d69a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,7 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
-#include "utils/string-utils.h"
+#include "utils/docker-util.h"
#include "util.h"
#include "config.h"
@@ -43,7 +43,6 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
-#include <regex.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -81,11 +80,6 @@ static const char* TC_READ_STATS_OPTS [] = { "-s", "-b", NULL};
//struct to store the user details
struct passwd *user_detail = NULL;
-//Docker container related constants.
-static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
-static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
-static const char* DOCKER_PULL_COMMAND = "pull";
-
FILE* LOGFILE = NULL;
FILE* ERRORFILE = NULL;
@@ -465,8 +459,9 @@ int is_feature_enabled(const char* feature_key, int default_value,
}
int is_docker_support_enabled() {
- return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
- DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg);
+ return is_feature_enabled(DOCKER_SUPPORT_ENABLED_KEY,
+ DEFAULT_DOCKER_SUPPORT_ENABLED, &executor_cfg)
+ || docker_module_enabled(&CFG);
}
int is_tc_support_enabled() {
@@ -474,13 +469,6 @@ int is_tc_support_enabled() {
DEFAULT_TC_SUPPORT_ENABLED, &executor_cfg);
}
-char* check_docker_binary(char *docker_binary) {
- if (docker_binary == NULL) {
- return "docker";
- }
- return docker_binary;
-}
-
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -1157,273 +1145,28 @@ int initialize_app(const char *user, const char *app_id,
return -1;
}
-static char* escape_single_quote(const char *str) {
- int p = 0;
- int i = 0;
- char replacement[] = "'\"'\"'";
- size_t replacement_length = strlen(replacement);
- size_t ret_size = strlen(str) * replacement_length + 1;
- char *ret = (char *) calloc(ret_size, sizeof(char));
- if(ret == NULL) {
- exit(OUT_OF_MEMORY);
- }
- while(str[p] != '\0') {
- if(str[p] == '\'') {
- strncat(ret, replacement, ret_size - strlen(ret));
- i += replacement_length;
- }
- else {
- ret[i] = str[p];
- ret[i + 1] = '\0';
- i++;
- }
- p++;
- }
- return ret;
-}
-
-static void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
- char *tmp = escape_single_quote(arg);
- strcat(*str, param);
- strcat(*str, "'");
- if(strlen(*str) + strlen(tmp) > *size) {
- *str = (char *) realloc(*str, strlen(*str) + strlen(tmp) + 1024);
- if(*str == NULL) {
- exit(OUT_OF_MEMORY);
- }
- *size = strlen(*str) + strlen(tmp) + 1024;
- }
- strcat(*str, tmp);
- strcat(*str, "' ");
- free(tmp);
-}
-
-char** tokenize_docker_command(const char *input, int *split_counter) {
- char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
- char **linesplit = (char **) malloc(sizeof(char *));
- char *p = NULL;
- *split_counter = 0;
- strncpy(line, input, strlen(input));
-
- p = strtok(line, " ");
- while(p != NULL) {
- linesplit[*split_counter] = p;
- (*split_counter)++;
- linesplit = realloc(linesplit, (sizeof(char *) * (*split_counter + 1)));
- if(linesplit == NULL) {
- fprintf(ERRORFILE, "Cannot allocate memory to parse docker command %s",
- strerror(errno));
- fflush(ERRORFILE);
- exit(OUT_OF_MEMORY);
- }
- p = strtok(NULL, " ");
- }
- linesplit[*split_counter] = NULL;
- return linesplit;
-}
-
-int execute_regex_match(const char *regex_str, const char *input) {
- regex_t regex;
- int regex_match;
- if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
- fprintf(LOGFILE, "Unable to compile regex.");
- fflush(LOGFILE);
- exit(ERROR_COMPILING_REGEX);
- }
- regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
- regfree(®ex);
- if(0 == regex_match) {
- return 0;
- }
- return 1;
-}
-
-int validate_docker_image_name(const char *image_name) {
- char *regex_str = "^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
- return execute_regex_match(regex_str, image_name);
-}
-
-char* sanitize_docker_command(const char *line) {
- static struct option long_options[] = {
- {"name", required_argument, 0, 'n' },
- {"user", required_argument, 0, 'u' },
- {"rm", no_argument, 0, 'r' },
- {"workdir", required_argument, 0, 'w' },
- {"net", required_argument, 0, 'e' },
- {"hostname", required_argument, 0, 'h' },
- {"cgroup-parent", required_argument, 0, 'g' },
- {"privileged", no_argument, 0, 'p' },
- {"cap-add", required_argument, 0, 'a' },
- {"cap-drop", required_argument, 0, 'o' },
- {"device", required_argument, 0, 'i' },
- {"detach", required_argument, 0, 't' },
- {"format", required_argument, 0, 'f' },
- {"group-add", required_argument, 0, 'x' },
- {0, 0, 0, 0}
- };
-
- int c = 0;
- int option_index = 0;
- char *output = NULL;
- size_t output_size = 0;
- char **linesplit;
- int split_counter = 0;
- int len = strlen(line);
-
- linesplit = tokenize_docker_command(line, &split_counter);
-
- output_size = len * 2;
- output = (char *) calloc(output_size, sizeof(char));
- if(output == NULL) {
- exit(OUT_OF_MEMORY);
- }
-
- // Handle docker client config option.
- if(0 == strncmp(linesplit[0], DOCKER_CLIENT_CONFIG_ARG, strlen(DOCKER_CLIENT_CONFIG_ARG))) {
- strcat(output, linesplit[0]);
- strcat(output, " ");
- long index = 0;
- while(index < split_counter) {
- linesplit[index] = linesplit[index + 1];
- if (linesplit[index] == NULL) {
- split_counter--;
- break;
- }
- index++;
- }
- }
-
- // Handle docker pull and image name validation.
- if (0 == strncmp(linesplit[0], DOCKER_PULL_COMMAND, strlen(DOCKER_PULL_COMMAND))) {
- if (0 != validate_docker_image_name(linesplit[1])) {
- fprintf(ERRORFILE, "Invalid Docker image name, exiting.");
- fflush(ERRORFILE);
- exit(DOCKER_IMAGE_INVALID);
- }
- strcat(output, linesplit[0]);
- strcat(output, " ");
- strcat(output, linesplit[1]);
- return output;
- }
-
- strcat(output, linesplit[0]);
- strcat(output, " ");
- optind = 1;
- while((c=getopt_long(split_counter, linesplit, "dv:", long_options, &option_index)) != -1) {
- switch(c) {
- case 'n':
- quote_and_append_arg(&output, &output_size, "--name=", optarg);
- break;
- case 'w':
- quote_and_append_arg(&output, &output_size, "--workdir=", optarg);
- break;
- case 'u':
- quote_and_append_arg(&output, &output_size, "--user=", optarg);
- break;
- case 'e':
- quote_and_append_arg(&output, &output_size, "--net=", optarg);
- break;
- case 'h':
- quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
- break;
- case 'v':
- quote_and_append_arg(&output, &output_size, "-v ", optarg);
- break;
- case 'a':
- quote_and_append_arg(&output, &output_size, "--cap-add=", optarg);
- break;
- case 'o':
- quote_and_append_arg(&output, &output_size, "--cap-drop=", optarg);
- break;
- case 'd':
- strcat(output, "-d ");
- break;
- case 'r':
- strcat(output, "--rm ");
- break;
- case 'g':
- quote_and_append_arg(&output, &output_size, "--cgroup-parent=", optarg);
- break;
- case 'p':
- strcat(output, "--privileged ");
- break;
- case 'i':
- quote_and_append_arg(&output, &output_size, "--device=", optarg);
- break;
- case 't':
- quote_and_append_arg(&output, &output_size, "--detach=", optarg);
- break;
- case 'f':
- strcat(output, "--format=");
- strcat(output, optarg);
- strcat(output, " ");
- break;
- case 'x':
- quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
- break;
- default:
- fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
- fflush(LOGFILE);
- return NULL;
- break;
- }
- }
-
- if(optind < split_counter) {
- while(optind < split_counter) {
- if (0 == strncmp(linesplit[optind], DOCKER_CONTAINER_NAME_PREFIX, strlen(DOCKER_CONTAINER_NAME_PREFIX))) {
- if (1 != validate_container_id(linesplit[optind])) {
- fprintf(ERRORFILE, "Specified container_id=%s is invalid\n", linesplit[optind]);
- fflush(ERRORFILE);
- exit(DOCKER_CONTAINER_NAME_INVALID);
- }
- strcat(output, linesplit[optind++]);
- } else {
- quote_and_append_arg(&output, &output_size, "", linesplit[optind++]);
- }
- }
- }
-
- return output;
-}
-
-char* parse_docker_command_file(const char* command_file) {
- size_t len = 0;
- char *line = NULL;
- ssize_t read;
- FILE *stream;
- stream = fopen(command_file, "r");
- if (stream == NULL) {
- fprintf(ERRORFILE, "Cannot open file %s - %s",
- command_file, strerror(errno));
- fflush(ERRORFILE);
- exit(ERROR_OPENING_DOCKER_FILE);
- }
- if ((read = getline(&line, &len, stream)) == -1) {
- fprintf(ERRORFILE, "Error reading command_file %s\n", command_file);
- fflush(ERRORFILE);
- exit(ERROR_READING_DOCKER_FILE);
- }
- fclose(stream);
-
- char* ret = sanitize_docker_command(line);
- if(ret == NULL) {
- exit(ERROR_SANITIZING_DOCKER_COMMAND);
+char *construct_docker_command(const char *command_file) {
+ int ret = 0;
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
+ char *buffer = alloc_and_clear_memory(command_size, sizeof(char));
+ ret = get_docker_command(command_file, &CFG, buffer, command_size);
+ if (ret != 0) {
+ fprintf(ERRORFILE, "Error constructing docker command, docker error code=%d, error message='%s'\n", ret,
+ get_docker_error_message(ret));
+ fflush(ERRORFILE);
+ exit(DOCKER_RUN_FAILED);
}
- fprintf(ERRORFILE, "Using command %s\n", ret);
- fflush(ERRORFILE);
-
- return ret;
+ return buffer;
}
int run_docker(const char *command_file) {
- char* docker_command = parse_docker_command_file(command_file);
- char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char* docker_command = construct_docker_command(command_file);
+ char* docker_binary = get_docker_binary(&CFG);
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- char* docker_command_with_binary = calloc(sizeof(char), command_size);
+ char* docker_command_with_binary = alloc_and_clear_memory(command_size, sizeof(char));
snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
+ fprintf(LOGFILE, "Invoking '%s'\n", docker_command_with_binary);
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
@@ -1437,8 +1180,9 @@ int run_docker(const char *command_file) {
free(docker_command_with_binary);
free(docker_command);
exit_code = DOCKER_RUN_FAILED;
+ } else {
+ exit_code = 0;
}
- exit_code = 0;
return exit_code;
}
@@ -1583,18 +1327,17 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- docker_command_with_binary = calloc(sizeof(char), command_size);
- docker_wait_command = calloc(sizeof(char), command_size);
- docker_logs_command = calloc(sizeof(char), command_size);
- docker_inspect_command = calloc(sizeof(char), command_size);
- docker_rm_command = calloc(sizeof(char), command_size);
+ docker_command_with_binary = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_wait_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_logs_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_inspect_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
+ docker_rm_command = (char *) alloc_and_clear_memory(command_size, sizeof(char));
gid_t user_gid = getegid();
uid_t prev_uid = geteuid();
- char *docker_command = parse_docker_command_file(command_file);
- char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
- docker_binary = check_docker_binary(docker_binary);
+ char *docker_command = NULL;
+ char *docker_binary = NULL;
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
@@ -1617,6 +1360,9 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
+ docker_command = construct_docker_command(command_file);
+ docker_binary = get_docker_binary(&CFG);
+
fprintf(LOGFILE, "Getting exit code file...\n");
exit_code_file = get_exit_code_file(pid_file);
if (NULL == exit_code_file) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index aa38abf..6d4220f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -59,16 +59,12 @@ enum operations {
#define MIN_USERID_KEY "min.user.id"
#define BANNED_USERS_KEY "banned.users"
#define ALLOWED_SYSTEM_USERS_KEY "allowed.system.users"
-#define DOCKER_BINARY_KEY "docker.binary"
#define DOCKER_SUPPORT_ENABLED_KEY "feature.docker.enabled"
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp"
extern struct passwd *user_detail;
-// get the executable's filename
-char* get_executable(char *argv0);
-
//function used to load the configurations present in the secure config
void read_executor_config(const char* file_name);
@@ -258,11 +254,6 @@ int is_docker_support_enabled();
*/
int run_docker(const char *command_file);
-/**
- * Sanitize docker commands. Returns NULL if there was any failure.
-*/
-char* sanitize_docker_command(const char *line);
-
/*
* Compile the regex_str and determine if the input string matches.
* Return 0 on match, 1 of non-match.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index 55973a2..e1ec293 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -29,12 +29,9 @@
*/
#include "config.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "util.h"
#include <errno.h>
-#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
new file mode 100644
index 0000000..ee6b375
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.h
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+#define __YARN_POSIX_CONTAINER_EXECUTOR_GET_EXECUTABLE_H__
+
+/**
+ * Get the path to executable that is currently running
+ * @param argv0 the name of the executable
+ * @return the path to the currently running executable
+ */
+char* get_executable(char *argv0);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index a05dc78..7b8f63f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -20,30 +20,15 @@
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
+#include "get_executable.h"
#include "modules/gpu/gpu-module.h"
#include "modules/cgroups/cgroups-operations.h"
#include <errno.h>
#include <grp.h>
-#include <limits.h>
#include <unistd.h>
-#include <signal.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/stat.h>
-
-#define CONF_FILENAME "container-executor.cfg"
-
-// When building as part of a Maven build this value gets defined by using
-// container-executor.conf.dir property. See:
-// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
-// for details.
-// NOTE: if this ends up being a relative path it gets resolved relative to
-// the location of the container-executor binary itself, not getwd(3)
-#ifndef HADOOP_CONF_DIR
- #error HADOOP_CONF_DIR must be defined
-#endif
static void display_usage(FILE *stream) {
fprintf(stream,
@@ -145,25 +130,21 @@ of whether an explicit checksetup operation is requested. */
static void assert_valid_setup(char *argv0) {
int ret;
char *executable_file = get_executable(argv0);
- if (!executable_file) {
- fprintf(ERRORFILE,"realpath of executable: %s\n",
- errno != 0 ? strerror(errno) : "unknown");
+ if (!executable_file || executable_file[0] == 0) {
+ fprintf(ERRORFILE, "realpath of executable: %s\n",
+ errno != 0 ? strerror(errno) : "unknown");
flush_and_close_log_files();
- exit(-1);
+ exit(INVALID_CONFIG_FILE);
}
- char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
- char *conf_file = resolve_config_path(orig_conf_file, executable_file);
+ char *conf_file = get_config_path(argv0);
if (conf_file == NULL) {
- free(executable_file);
- fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
- free(executable_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
index f0c6d16..4c76e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
@@ -16,9 +16,8 @@
* limitations under the License.
*/
+#include "module-configs.h"
#include "util.h"
-#include "configuration.h"
-#include "container-executor.h"
#include "modules/common/constants.h"
#include <string.h>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
index d58c618..b8c57df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
@@ -23,6 +23,7 @@
#ifndef _MODULES_COMMON_MODULE_CONFIGS_H_
#define _MODULES_COMMON_MODULE_CONFIGS_H_
+#include "configuration.h"
/**
* check if module enabled given name of module.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
index 8e39ca8..a9539cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -17,10 +17,10 @@
*/
#include "util.h"
-#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
+#include <regex.h>
char** split_delimiter(char *value, const char *delim) {
char **return_values = NULL;
@@ -132,3 +132,61 @@ char* trim(const char* input) {
ret[val_end - val_begin] = '\0';
return ret;
}
+
+int execute_regex_match(const char *regex_str, const char *input) {
+ regex_t regex;
+ int regex_match;
+ if (0 != regcomp(®ex, regex_str, REG_EXTENDED|REG_NOSUB)) {
+ fprintf(LOGFILE, "Unable to compile regex.");
+ fflush(LOGFILE);
+ exit(ERROR_COMPILING_REGEX);
+ }
+ regex_match = regexec(®ex, input, (size_t) 0, NULL, 0);
+ regfree(®ex);
+ if(0 == regex_match) {
+ return 0;
+ }
+ return 1;
+}
+
+char* escape_single_quote(const char *str) {
+ int p = 0;
+ int i = 0;
+ char replacement[] = "'\"'\"'";
+ size_t replacement_length = strlen(replacement);
+ size_t ret_size = strlen(str) * replacement_length + 1;
+ char *ret = (char *) alloc_and_clear_memory(ret_size, sizeof(char));
+ if(ret == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ while(str[p] != '\0') {
+ if(str[p] == '\'') {
+ strncat(ret, replacement, ret_size - strlen(ret));
+ i += replacement_length;
+ }
+ else {
+ ret[i] = str[p];
+ i++;
+ }
+ p++;
+ }
+ ret[i] = '\0';
+ return ret;
+}
+
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg) {
+ char *tmp = escape_single_quote(arg);
+ int alloc_block = 1024;
+ strcat(*str, param);
+ strcat(*str, "'");
+ if (strlen(*str) + strlen(tmp) > *size) {
+ *size = (strlen(*str) + strlen(tmp) + alloc_block) * sizeof(char);
+ *str = (char *) realloc(*str, *size);
+ if (*str == NULL) {
+ exit(OUT_OF_MEMORY);
+ }
+ }
+ strcat(*str, tmp);
+ strcat(*str, "' ");
+ free(tmp);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3b1c631/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index fa21def..8758d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -19,7 +19,11 @@
#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
+/** Define a platform-independent constant instead of using PATH_MAX, set to 4K */
+#define EXECUTOR_PATH_MAX 4096
+
#include <stdio.h>
+#include <stdlib.h>
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
@@ -62,7 +66,7 @@ enum errorcodes {
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
- DOCKER_CONTAINER_NAME_INVALID = 41,
+ // DOCKER_CONTAINER_NAME_INVALID = 41, (NOT USED)
ERROR_COMPILING_REGEX = 42
};
@@ -119,4 +123,44 @@ void free_values(char **values);
*/
char* trim(const char *input);
+/**
+ * Run a regex to check if the provided input matches against it
+ * @param regex_str Regex to run
+ * @param input String to run the regex against
+ * @return 0 on match, non-zero on no match
+ */
+int execute_regex_match(const char *regex_str, const char *input);
+
+/**
+ * Helper function to escape single-quotes in a string. The assumption is that the string passed will be enclosed in
+ * single quotes and passed to bash for a command invocation.
+ * @param str The string in which to esacpe single quotes
+ * @return String with single quotes escaped, must be freed by the user.
+ */
+char* escape_single_quote(const char *str);
+
+/**
+ * Helper function to quote the argument to a parameter and then append it to the provided string.
+ * @param str Buffer to which the param='arg' string must be appended
+ * @param size Size of the buffer
+ * @param param Param name
+ * @param arg Argument to be quoted
+ */
+void quote_and_append_arg(char **str, size_t *size, const char* param, const char *arg);
+
+/**
+ * Helper function to allocate and clear a block of memory. It'll call exit if the allocation fails.
+ * @param num Num of elements to be allocated
+ * @param size Size of each element
+ * @return Pointer to the allocated memory, must be freed by the user
+ */
+inline void* alloc_and_clear_memory(size_t num, size_t size) {
+ void *ret = calloc(num, size);
+ if (ret == NULL) {
+ printf("Could not allocate memory, exiting\n");
+ exit(OUT_OF_MEMORY);
+ }
+ return ret;
+}
+
#endif
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org