You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vv...@apache.org on 2016/01/22 10:18:34 UTC
[2/2] hadoop git commit: YARN-4578. Directories that are mounted in
docker containers need to be more restrictive/container-specific. Contributed
by Sidharta Seethana.
YARN-4578. Directories that are mounted in docker containers need to be more restrictive/container-specific. Contributed by Sidharta Seethana.
(cherry picked from commit b41a7e89d1eb8650975ac7092532ed9563ac60f2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85561207
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85561207
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85561207
Branch: refs/heads/branch-2
Commit: 85561207f986c1616b48e9dea9251ebd0b433e3e
Parents: 73ef77c
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Jan 22 14:43:14 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Jan 22 14:44:22 2016 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/LinuxContainerExecutor.java | 4 ++++
.../launcher/ContainerLaunch.java | 11 +++++++++
.../runtime/DockerLinuxContainerRuntime.java | 10 ++++++--
.../runtime/LinuxContainerRuntimeConstants.java | 4 ++++
.../executor/ContainerStartContext.java | 24 ++++++++++++++++++++
.../runtime/TestDockerContainerRuntime.java | 12 ++++++++--
7 files changed, 64 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 49bcb73..3a09562 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,9 @@ Release 2.9.0 - UNRELEASED
YARN-4584. RM startup failure when AM attempts greater than max-attempts.
(Bibin A Chundatt via rohithsharmaks)
+ YARN-4578. Directories that are mounted in docker containers need to be more
+ restrictive/container-specific. (Sidharta Seethana via vvasudev)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 8549230..ff82e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -291,6 +291,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
Path containerWorkDir = ctx.getContainerWorkDir();
List<String> localDirs = ctx.getLocalDirs();
List<String> logDirs = ctx.getLogDirs();
+ List<String> containerLocalDirs = ctx.getContainerLocalDirs();
+ List<String> containerLogDirs = ctx.getContainerLogDirs();
Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
verifyUsernamePattern(user);
@@ -375,6 +377,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+ .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
if (tcCommandFile != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 6371b21..64689dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -253,6 +253,7 @@ public class ContainerLaunch implements Callable<Integer> {
+ dirsHandler.getDisksHealthReport(false));
}
+ List<String> containerLocalDirs = new ArrayList<>(localDirs.size());
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
@@ -261,6 +262,14 @@ public class ContainerLaunch implements Callable<Integer> {
Path userdir = new Path(usersdir, user);
Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
appDirs.add(new Path(appsdir, appIdStr));
+
+ String containerLocalDir = localDir + Path.SEPARATOR +
+ ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+ + Path.SEPARATOR
+ + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+ + Path.SEPARATOR;
+
+ containerLocalDirs.add(containerLocalDir);
}
containerScriptOutStream =
lfs.create(nmPrivateContainerScriptPath,
@@ -317,6 +326,8 @@ public class ContainerLaunch implements Callable<Integer> {
.setContainerWorkDir(containerWorkDir)
.setLocalDirs(localDirs)
.setLogDirs(logDirs)
+ .setContainerLocalDirs(containerLocalDirs)
+ .setContainerLogDirs(containerLogDirs)
.build());
}
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index f8aae81..2dee663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -223,6 +223,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> containerLocalDirs = ctx.getExecutionAttribute(
+ CONTAINER_LOCAL_DIRS);
+ @SuppressWarnings("unchecked")
+ List<String> containerLogDirs = ctx.getExecutionAttribute(
+ CONTAINER_LOG_DIRS);
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -235,10 +241,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.setNetworkType("host")
.setCapabilities(capabilities)
.addMountLocation("/etc/passwd", "/etc/password:ro");
- List<String> allDirs = new ArrayList<>(localDirs);
+ List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.add(containerWorkDir.toString());
- allDirs.addAll(logDirs);
+ allDirs.addAll(containerLogDirs);
for (String dir: allDirs) {
runCommand.addMountLocation(dir, dir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
index d2069a9..96ff40d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
@@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants {
List.class, "local_dirs");
public static final Attribute<List> LOG_DIRS = Attribute.attribute(
List.class, "log_dirs");
+ public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
+ .attribute(List.class, "container_local_dirs");
+ public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(
+ List.class, "container_log_dirs");
public static final Attribute<String> RESOURCES_OPTIONS = Attribute.attribute(
String.class, "resources_options");
public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
index ffcc519..b14a520 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
@@ -45,6 +45,8 @@ public final class ContainerStartContext {
private final Path containerWorkDir;
private final List<String> localDirs;
private final List<String> logDirs;
+ private final List<String> containerLocalDirs;
+ private final List<String> containerLogDirs;
public static final class Builder {
private Container container;
@@ -56,6 +58,8 @@ public final class ContainerStartContext {
private Path containerWorkDir;
private List<String> localDirs;
private List<String> logDirs;
+ private List<String> containerLocalDirs;
+ private List<String> containerLogDirs;
public Builder() {
}
@@ -107,6 +111,16 @@ public final class ContainerStartContext {
return this;
}
+ public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
+ this.containerLocalDirs = containerLocalDirs;
+ return this;
+ }
+
+ public Builder setContainerLogDirs(List<String> containerLogDirs) {
+ this.containerLogDirs = containerLogDirs;
+ return this;
+ }
+
public ContainerStartContext build() {
return new ContainerStartContext(this);
}
@@ -122,6 +136,8 @@ public final class ContainerStartContext {
this.containerWorkDir = builder.containerWorkDir;
this.localDirs = builder.localDirs;
this.logDirs = builder.logDirs;
+ this.containerLocalDirs = builder.containerLocalDirs;
+ this.containerLogDirs = builder.containerLogDirs;
}
public Container getContainer() {
@@ -163,4 +179,12 @@ public final class ContainerStartContext {
public List<String> getLogDirs() {
return Collections.unmodifiableList(this.logDirs);
}
+
+ public List<String> getContainerLocalDirs() {
+ return this.containerLocalDirs;
+ }
+
+ public List<String> getContainerLogDirs() {
+ return this.containerLogDirs;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85561207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 7fc0158..6898634 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -81,6 +81,8 @@ public class TestDockerContainerRuntime {
Path pidFilePath;
List<String> localDirs;
List<String> logDirs;
+ List<String> containerLocalDirs;
+ List<String> containerLogDirs;
String resourcesOptions;
ContainerRuntimeContext.Builder builder;
String submittingUser = "anakin";
@@ -123,9 +125,13 @@ public class TestDockerContainerRuntime {
localDirs = new ArrayList<>();
logDirs = new ArrayList<>();
resourcesOptions = "cgroups=none";
+ containerLocalDirs = new ArrayList<>();
+ containerLogDirs = new ArrayList<>();
localDirs.add("/test_local_dir");
logDirs.add("/test_log_dir");
+ containerLocalDirs.add("/test_container_local_dir");
+ containerLogDirs.add("/test_container_log_dir");
builder = new ContainerRuntimeContext
.Builder(container);
@@ -141,6 +147,8 @@ public class TestDockerContainerRuntime {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
+ .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+ .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
}
@@ -245,8 +253,8 @@ public class TestDockerContainerRuntime {
.append("bash %8$s/launch_container.sh");
String expectedCommand = String.format(expectedCommandTemplate.toString(),
- containerId, runAsUser, containerWorkDir, localDirs.get(0),
- containerWorkDir, logDirs.get(0), image, containerWorkDir);
+ containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
+ containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));