You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@myriad.apache.org by da...@apache.org on 2016/05/12 04:02:22 UTC
incubator-myriad git commit: Created better cgroup support please see
cgroups.md for documentation. JIRA: [Myriad-192]
https://issues.apache.org/jira/browse/MYRIAD-192 Pull Request: Closes #69
Author: DarinJ
Repository: incubator-myriad
Updated Branches:
refs/heads/master 5118cff3e -> fe493af32
Created better cgroup support please see cgroups.md for documentation.
JIRA:
[Myriad-192] https://issues.apache.org/jira/browse/MYRIAD-192
Pull Request:
Closes #69
Author:
DarinJ <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/fe493af3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/fe493af3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/fe493af3
Branch: refs/heads/master
Commit: fe493af32eeee9e069f9c228c8847545d31a47be
Parents: 5118cff
Author: darinj <da...@gmail.com>
Authored: Fri May 6 00:02:57 2016 -0400
Committer: darinj <da...@apache.org>
Committed: Wed May 11 23:56:19 2016 -0400
----------------------------------------------------------------------
docs/cgroups.md | 49 +++++++++-----
.../configuration/MyriadConfiguration.java | 8 ++-
.../MyriadExecutorConfiguration.java | 1 +
.../scheduler/DownloadNMExecutorCLGenImpl.java | 22 +++----
.../myriad/scheduler/NMExecutorCLGenImpl.java | 67 ++++++++++----------
.../scheduler/ServiceCommandLineGenerator.java | 2 +-
.../scheduler/TestServiceCommandLine.java | 13 ++--
7 files changed, 90 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/docs/cgroups.md
----------------------------------------------------------------------
diff --git a/docs/cgroups.md b/docs/cgroups.md
index 47228bb..8694db8 100644
--- a/docs/cgroups.md
+++ b/docs/cgroups.md
@@ -30,18 +30,35 @@ Enabling Cgroups for YARN NodeManager involves:
* Modifying the $YARN_HOME/etc/hadoop/myriad-config-default.yml file.
* Modifying the $YARN_HOME/etc/hadoop/yarn-site.xml file.
+### Modify container-executor.cfg
+```
+yarn.nodemanager.linux-container-executor.group=yarn #should match yarn.nodemanager.linux-container-executor.group in yarn-site.xml
+banned.users=
+min.user.id=1000
+```
+### Verify Permissions
+
+The paths to container-executor.cfg and container-executor must be owned and writable only by root. The container-executor
+should have user-ownership by root and group ownership by the user running YARN (often yarn or hduser), which should match the
+yarn.nodemanager.linux-container-executor.group in yarn-site.xml and yarn.nodemanager.linux-container-executor.group in
+container-executor.cfg. Further the permission of container-executor should be r-Sr-s---.
+```
+chmod 6050 container-executor
+```
+If using remote distribution be sure to use the -p option of tar (as root) to perserve the suid bit.
+
### Modify Myriad-Config-default.yml ###
Modify the $YARN_HOME/etc/hadoop/myriad-config-default.yml file by adding the following content:
```
...
+frameworkSuperUser: root # Must be root or have passwordless sudo.
nodemanager:
-cgroups: true
+ cgroupPath: /path/to/cgroup # default is /sys/fs/cgroup
...
```
-
### Modify yarn-site.yml
Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by adding the following content:
@@ -50,33 +67,33 @@ Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by adding the following co
<property>
<description>who will execute(launch) the containers.</description>
<name>yarn.nodemanager.container-executor.class</name>
-<value>${yarn.nodemanager.container-executor.class}</value>
+<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
</property>
<property>
<description>The class which should help the LCE handle resources.</description>
<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
-<value>${yarn.nodemanager.linux-container-executor.resources-handler.class}</value>
+<value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
</property>
<property>
-<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
-<value>${yarn.nodemanager.linux-container-executor.cgroups.hierarchy}</value>
-</property>
-<property>
-<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
-<value>${yarn.nodemanager.linux-container-executor.cgroups.mount}</value>
+<name>yarn.nodemanager.linux-container-executor.group</name>
+<value>yarn</value>
</property>
<property>
-<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
-<value>${yarn.nodemanager.linux-container-executor.cgroups.mount-path}</value>
+<name>yarn.nodemanager.linux-container-executor.path</name>
+<value>${yarn.home}/bin/container-executor</value>
</property>
+
+<!-- Optional parameters, usually unnecessary
<property>
-<name>yarn.nodemanager.linux-container-executor.group</name>
-<value>${yarn.nodemanager.linux-container-executor.group}</value>
+<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+<value>true</value>
</property>
<property>
-<name>yarn.nodemanager.linux-container-executor.path</name>
-<value>${yarn.home}/bin/container-executor</value>
+<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+<value>/sys/fs/cgroup</value>
+<description>/sys/fs/cgroup and /cgroup are most common values</description>
</property>
+-->
```
---
<sub>
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
index f65bb9a..3de72a6 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
@@ -50,7 +50,7 @@ import org.hibernate.validator.constraints.NotEmpty;
* jvmMaxMemoryMB: 1024
* user: hduser
* cpus: 0.2
- * cgroups: false
+ * cgroupPath: /sys/fs/cgroup
* executor:
* jvmMaxMemoryMB: 256
* path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
@@ -185,6 +185,9 @@ public class MyriadConfiguration {
@JsonProperty
private String servedBinaryPath;
+ @JsonProperty
+ private String cgroupPath;
+
public MyriadConfiguration() {
}
@@ -291,4 +294,7 @@ public class MyriadConfiguration {
return Optional.fromNullable(servedBinaryPath);
}
+ public String getCGroupPath() {
+ return cgroupPath == null ? "/sys/fs/cgroup" : cgroupPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
index ea98ef7..1096e16 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java
@@ -77,4 +77,5 @@ public class MyriadExecutorConfiguration {
public Optional<String> getJvmUri() {
return Optional.fromNullable(jvmUri);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java
index 67fd1ae..74deda3 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/DownloadNMExecutorCLGenImpl.java
@@ -44,12 +44,11 @@ public class DownloadNMExecutorCLGenImpl extends NMExecutorCLGenImpl {
public String generateCommandLine(ServiceResourceProfile profile, Ports ports) {
StringBuilder cmdLine = new StringBuilder();
LOGGER.info("Using remote distribution");
-
generateEnvironment(profile, (NMPorts) ports);
appendDistroExtractionCommands(cmdLine);
appendCgroupsCmds(cmdLine);
appendYarnHomeExport(cmdLine);
- appendUser(cmdLine);
+ appendUserSudo(cmdLine);
appendEnvForNM(cmdLine);
cmdLine.append(YARN_NM_CMD);
return cmdLine.toString();
@@ -65,21 +64,16 @@ public class DownloadNMExecutorCLGenImpl extends NMExecutorCLGenImpl {
//TODO(DarinJ) support other compression, as this is a temp fix for Mesos 1760 may not get to it.
//Extract tarball keeping permissions, necessary to keep HADOOP_HOME/bin/container-executor suidbit set.
- cmdLine.append("sudo tar -zxpf ").append(getFileName(nodeManagerUri));
-
- //We need the current directory to be writable by frameworkUser for capsuleExecutor to create directories.
- //Best to simply give owenership to the user running the executor but we don't want to use -R as this
- //will silently remove the suid bit on container executor.
- cmdLine.append(" && sudo chown ").append(cfg.getFrameworkUser().get()).append(" .");
-
+ appendSudo(cmdLine);
+ cmdLine.append("tar -zxpf ").append(getFileName(nodeManagerUri));
//Place the hadoop config where in the HADOOP_CONF_DIR where it will be read by the NodeManager
//The url for the resource manager config is: http(s)://hostname:port/conf so fetcher.cpp downloads the
//config file to conf, It's an xml file with the parameters of yarn-site.xml, core-site.xml and hdfs.xml.
- cmdLine.append(" && cp conf ").append(cfg.getYarnEnvironment().get("YARN_HOME")).append("/etc/hadoop/yarn-site.xml;");
- }
-
- protected void appendUser(StringBuilder cmdLine) {
- cmdLine.append(" sudo -E -u ").append(cfg.getFrameworkUser().get()).append(" -H");
+ cmdLine.append(" && ");
+ appendSudo(cmdLine);
+ cmdLine.append(" cp conf ");
+ cmdLine.append(cfg.getYarnEnvironment().get("YARN_HOME"));
+ cmdLine.append("/etc/hadoop/yarn-site.xml;");
}
private static String getFileName(String uri) {
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
index 19eda34..2700488 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
@@ -38,27 +38,10 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator {
public static final String KEY_YARN_RM_HOSTNAME = "yarn.resourcemanager.hostname";
/**
- * YARN container executor class.
- */
- public static final String KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS = "yarn.nodemanager.container-executor.class";
- // TODO (mohit): Should it be configurable ?
- public static final String VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS =
- "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
- public static final String DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS =
- "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor";
- /**
* YARN class to help handle LCE resources
*/
- public static final String KEY_YARN_NM_LCE_RH_CLASS = "yarn.nodemanager.linux-container-executor.resources-handler.class";
// TODO (mohit): Should it be configurable ?
- public static final String VAL_YARN_NM_LCE_RH_CLASS = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
- public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierarchy";
- public static final String VAL_YARN_NM_LCE_CGROUPS_HIERARCHY = "mesos/$TASK_DIR";
- public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT = "yarn.nodemanager.linux-container-executor.cgroups.mount";
- public static final String KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path";
- public static final String VAL_YARN_NM_LCE_CGROUPS_MOUNT_PATH = "/sys/fs/cgroup";
- public static final String KEY_YARN_NM_LCE_GROUP = "yarn.nodemanager.linux-container-executor.group";
- public static final String KEY_YARN_NM_LCE_PATH = "yarn.nodemanager.linux-container-executor.path";
+ public static final String KEY_YARN_NM_LCE_CGROUPS_HIERARCHY = "yarn.nodemanager.linux-container-executor.cgroups.hierachy";
public static final String KEY_YARN_HOME = "yarn.home";
public static final String KEY_NM_RESOURCE_CPU_VCORES = "nodemanager.resource.cpu-vcores";
public static final String KEY_NM_RESOURCE_MEM_MB = "nodemanager.resource.memory-mb";
@@ -73,6 +56,7 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator {
private Map<String, String> environment = new HashMap<>();
protected MyriadConfiguration cfg;
+ protected YarnConfiguration conf = new YarnConfiguration();
public NMExecutorCLGenImpl(MyriadConfiguration cfg) {
this.cfg = cfg;
@@ -103,21 +87,10 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator {
}
if (cfg.getNodeManagerConfiguration().getCgroups().or(Boolean.FALSE)) {
- addYarnNodemanagerOpt(KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS, VAL_YARN_NM_CONTAINER_EXECUTOR_CLASS);
- addYarnNodemanagerOpt(KEY_YARN_NM_LCE_RH_CLASS, VAL_YARN_NM_LCE_RH_CLASS);
-
- // TODO: Configure hierarchy
- addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_HIERARCHY, VAL_YARN_NM_LCE_CGROUPS_HIERARCHY);
- addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_MOUNT, "true");
- // TODO: Make it configurable
- addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_MOUNT_PATH, VAL_YARN_NM_LCE_CGROUPS_MOUNT_PATH);
- addYarnNodemanagerOpt(KEY_YARN_NM_LCE_GROUP, "root");
+ addYarnNodemanagerOpt(KEY_YARN_NM_LCE_CGROUPS_HIERARCHY, "mesos/$TASK_DIR");
if (environment.containsKey("YARN_HOME")) {
addYarnNodemanagerOpt(KEY_YARN_HOME, environment.get("YARN_HOME"));
}
- } else {
- // Otherwise configure to use Default
- addYarnNodemanagerOpt(KEY_YARN_NM_CONTAINER_EXECUTOR_CLASS, DEFAULT_YARN_NM_CONTAINER_EXECUTOR_CLASS);
}
addYarnNodemanagerOpt(KEY_NM_RESOURCE_CPU_VCORES, Integer.toString(profile.getCpus().intValue()));
addYarnNodemanagerOpt(KEY_NM_RESOURCE_MEM_MB, Integer.toString(profile.getMemory().intValue()));
@@ -135,15 +108,40 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator {
}
protected void appendCgroupsCmds(StringBuilder cmdLine) {
- if (cfg.getNodeManagerConfiguration().getCgroups().or(Boolean.FALSE)) {
- cmdLine.append(" export TASK_DIR=`basename $PWD`;");
- cmdLine.append(" chmod +x /sys/fs/cgroup/cpu/mesos/$TASK_DIR;");
+ if (cfg.getFrameworkSuperUser().isPresent()) {
+ cmdLine.append(" export TASK_DIR=`basename $PWD`&&");
+ appendSudo(cmdLine);
+ //The container executor script expects mount-path to exist and owned by the yarn user
+ //See: https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html
+ //If YARN ever moves to cgroup/mem it will be necessary to add a mem version.
+ appendSudo(cmdLine);
+ cmdLine.append("chown " + cfg.getFrameworkUser().get() + " ");
+ cmdLine.append(cfg.getCGroupPath());
+ cmdLine.append("/cpu/mesos/$TASK_DIR &&");
+ } else {
+ LOGGER.info("frameworkSuperUser not enabled ignoring cgroup configuration");
}
}
protected void appendYarnHomeExport(StringBuilder cmdLine) {
if (environment.containsKey("YARN_HOME")) {
- cmdLine.append(" export YARN_HOME=" + environment.get("YARN_HOME") + ";");
+ cmdLine.append(" export YARN_HOME=");
+ cmdLine.append(environment.get("YARN_HOME"));
+ cmdLine.append(";");
+ }
+ }
+
+ protected void appendSudo(StringBuilder cmdLine) {
+ if (cfg.getFrameworkSuperUser().isPresent()) {
+ cmdLine.append(" sudo ");
+ }
+ }
+
+ protected void appendUserSudo(StringBuilder cmdLine) {
+ if (cfg.getFrameworkSuperUser().isPresent()) {
+ cmdLine.append(" sudo -E -u ");
+ cmdLine.append(cfg.getFrameworkUser().get());
+ cmdLine.append(" -H ");
}
}
@@ -159,7 +157,6 @@ public class NMExecutorCLGenImpl implements ExecutorCommandLineGenerator {
@Override
public String getConfigurationUrl() {
- YarnConfiguration conf = new YarnConfiguration();
String httpPolicy = conf.get(TaskFactory.YARN_HTTP_POLICY);
if (httpPolicy != null && httpPolicy.equals(TaskFactory.YARN_HTTP_POLICY_HTTPS_ONLY)) {
String address = conf.get(TaskFactory.YARN_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS);
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java
index 6fd8872..8765226 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceCommandLineGenerator.java
@@ -33,7 +33,7 @@ public class ServiceCommandLineGenerator extends DownloadNMExecutorCLGenImpl {
public String generateCommandLine(ServiceResourceProfile profile, Ports ports) {
StringBuilder strB = new StringBuilder();
appendDistroExtractionCommands(strB);
- appendUser(strB);
+ appendUserSudo(strB);
return strB.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fe493af3/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
index ab7e7d9..e49c19c 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
@@ -37,12 +37,12 @@ public class TestServiceCommandLine {
static MyriadConfiguration cfg;
static String toJHSCompare =
- "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " +
- "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp" +
+ "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " +
+ "export TASK_DIR=`basename $PWD`; sudo chmod +wx /sys/fs/cgroup/cpu/mesos/$TASK_DIR;" +
+ "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\"; sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo cp" +
" conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver";
-
static String toCompare =
- "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+ "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;";
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -63,6 +63,8 @@ public class TestServiceCommandLine {
ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0);
CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
+ System.out.println(toJHSCompare);
+ System.out.println(cInfo.getValue());
assertTrue(cInfo.getValue().startsWith(toCompare));
}
@@ -79,7 +81,8 @@ public class TestServiceCommandLine {
NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator);
CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts);
-
+ System.out.println(toCompare);
+ System.out.println(cInfo.getValue());
assertTrue(cInfo.getValue().startsWith(toCompare));
}