You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@myriad.apache.org by da...@apache.org on 2016/05/12 01:25:27 UTC
incubator-myriad git commit: Added Docker Support,
see the README.md in docker directory for quick start guide.
Repository: incubator-myriad
Updated Branches:
refs/heads/master 519d9b421 -> bcd1fc8c6
Added Docker Support, see the README.md in docker directory for quick start guide.
Code changes:
- Added classes necessary to pass configuration to TaskFactory
- Modified classes necessary to consume the configuration changes
- Edits to docker/scripts mostly to break up the build
@Todo
- add support for MesosInfo (for AppC support), will be done as more information about the MesosInfo object appears.
- add configuration Uri to avoid rebuilding images (done, separate PR).
- clean up docker scripts more, including the ability to pass build arguments, will be incrementally done.
JIRA:
[Myriad-136] https://issues.apache.org/jira/browse/MYRIAD-136
Pull Request:
Closes #67
Author:
DarinJ <da...@apache.org>
Code changes:
- Added classes necessary to pass configuration to TaskFactory
- Modified classes necessary to consume the configuration changes
- Edits to docker/scripts mostly to break up the build
@Todo
- add support for MesosInfo (for AppC support), will be done as more information about the MesosInfo object appears.
- add configuration Uri to avoid rebuilding images (done, separate PR).
- clean up docker scripts more, including the ability to pass build arguments, will be incrementally done.
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/bcd1fc8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/bcd1fc8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/bcd1fc8c
Branch: refs/heads/master
Commit: bcd1fc8c6f6ea184522bc9a2c58610710a4be9b3
Parents: 519d9b4
Author: darinj <da...@gmail.com>
Authored: Mon May 2 15:08:20 2016 -0400
Committer: darinj <da...@apache.org>
Committed: Wed May 11 21:18:36 2016 -0400
----------------------------------------------------------------------
build.gradle | 8 +
docker/Dockerfile | 21 ++-
docker/README.md | 29 ++--
docker/build-myriad.sh | 3 +-
docker/build.gradle | 18 ++-
docker/myriad-bin/configure-yarn.sh | 60 +++++++
docker/myriad-bin/create-user.sh | 34 ----
docker/myriad-bin/install-yarn.sh | 54 ++-----
docker/myriad-etc/mapred-site.xml.template | 29 ++++
.../myriad-config-default.yml.template | 113 +++++++++++++
docker/myriad-etc/yarn-site.xml.template | 162 +++++++++++++++++++
myriad-scheduler/build.gradle | 1 -
.../configuration/MyriadConfiguration.java | 30 +++-
.../MyriadContainerConfiguration.java | 54 +++++++
.../MyriadDockerConfiguration.java | 72 +++++++++
.../scheduler/ServiceTaskFactoryImpl.java | 3 +
.../apache/myriad/scheduler/TaskFactory.java | 11 +-
.../org/apache/myriad/scheduler/TaskUtils.java | 95 ++++++++++-
.../apache/myriad/scheduler/TestTaskUtils.java | 94 ++++++++++-
...iad-config-test-default-with-docker-info.yml | 90 +++++++++++
...-config-test-default-with-framework-role.yml | 70 ++++++++
21 files changed, 934 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 54e3eac..ac416ec 100644
--- a/build.gradle
+++ b/build.gradle
@@ -77,6 +77,13 @@ subprojects {
configurations.create('myriadExecutorConf')
+ //Keeping guava at the level of hadoop 2.6.0 so we don't get all the
+ //awesomeness in our IDE and fail at runtime.
+ configurations.all {
+ resolutionStrategy {
+ force 'com.google.guava:guava:11.0.2'
+ }
+ }
configurations {
provided
capsule
@@ -97,6 +104,7 @@ subprojects {
compile 'com.google.code.gson:gson:2.3.1' // marshalling between the scheduler and executor
testCompile 'org.apache.zookeeper:zookeeper:3.4.6' // to resolve temporary mavenlocal issue
+ runtime 'com.google.guava:guava:11.0.2'
testCompile 'junit:junit:4.12'
testCompile 'commons-collections:commons-collections:3.2.1'
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 52ad7f3..4f344a6 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -24,6 +24,10 @@ MAINTAINER Apache Myriad dev@myriad.incubator.apache.org
ENV HADOOP_USER="yarn"
ENV HADOOP_VER="2.7.0"
+ENV HADOOP_HOME=/usr/local/hadoop
+ENV JAVA_HOME=/usr
+ENV MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so
+
# Setup mesosphere repositories
RUN apt-get -y update
@@ -32,24 +36,17 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) && echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list
RUN apt-get -y update
-ADD myriad-bin/create-user.sh /create-user.sh
-RUN sh /create-user.sh
-
# Install Mesos
RUN apt-get install -y mesos curl tar
-
# Run local YARN install
ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
-
+ADD myriad-bin/configure-yarn.sh /configure-yarn.sh
+RUN /configure-yarn.sh
# Copy over myriad libraries
ADD ./libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
-
-
# Initialize hadoop confs with env vars
-ADD myriad-bin/run-myriad.sh /run-myriad.sh
-RUN mkdir /myriad-conf/
-
-# Run the YARN resource manager
+ADD myriad-etc/* /usr/local/hadoop/etc/hadoop/
+RUN chown -R root:root /usr/local/hadoop/etc
USER yarn
-CMD sh /run-myriad.sh
+CMD /usr/local/hadoop/bin/yarn resourcemanager
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/README.md
----------------------------------------------------------------------
diff --git a/docker/README.md b/docker/README.md
index 7ce77ac..1650691 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -2,25 +2,36 @@
Docker is the easiest way to from 0 to Yarn on Mesos within minutes.
## Building the Resource Manager Docker
-Run the following command:
-```./gradlew buildRMDocker```
-This will build the ResourceManager from src and save the image as *mesos/myriad*.
#Configuration Guide#
-In order for the ResourceManager to operate correctly, you will need to provide 2 configuration files. These files will need to mounted from a directory into */myriad-conf* within the docker container.
+In order for the ResourceManager to operate correctly, you will need to provide 5 configuration files. These files will
+need to mounted from a directory into `./myriad-etc` of this folder.
-* [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
-* modified [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md)
+* [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml) (template provided)
+* [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md) (template provided)
+* mapred-site.xml (template provided)
+* hdfs-site.xml (used for hdfs)
+* core-site.xml (used for hdfs)
+It is assumed you already have hdfs and Mesos running. For more information about Apache Mesos visit the [website](http://mesos.apache.org).
+If you need to setup hdfs, consider using the [hdfs-mesos framework](https://github.com/mesosphere/hdfs). Copy the hdfs.xml and
+core-site.xml from your hdfs configuration into myriad-etc. Additional, files maybe necessary such as rack.sh, log4j.properties,
+hadoop-env.sh, and yarn-env.sh depending on your configuration.
+
+Run the following commands:
+```bash
+./gradlew -P username/myriad buildRMDocker
+docker push username/myriad
+```
+This will build the ResourceManager from src, save, and push the image as *username/myriad*.
## Running the Resource Manager Docker
```bash
docker run --net=host --name='myriad-resourcemanager' -t \
- -v /path/to/configs:/myriad-conf \
- mesos/myriad
- ```
+ <username>/myriad
+```
#Environment Variables#
* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used by Myriad. This will download the *.tar.gz file to be used as the hadoop distribution of choice for Myriad.
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/build-myriad.sh
----------------------------------------------------------------------
diff --git a/docker/build-myriad.sh b/docker/build-myriad.sh
index d3b0e74..b0638d0 100644
--- a/docker/build-myriad.sh
+++ b/docker/build-myriad.sh
@@ -36,6 +36,7 @@ if [ ! -d "libs" ]; then
fi
# Copy over the Java Libraries
-cp -rf ../myriad-scheduler/build/libs/* libs/
+cp -rf ../myriad-scheduler/build/libs/* libs/
+cp -rf ../myriad-executor/build/libs/* libs/
echo "Init complete! " #Modify config/myriad-default-config.yml to your liking before building the docker image"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/build.gradle
----------------------------------------------------------------------
diff --git a/docker/build.gradle b/docker/build.gradle
index 4f35494..38f61e6 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -23,15 +23,25 @@ import com.bmuschko.gradle.docker.tasks.image.Dockerfile
import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
task copySchedulerJars(type:Copy){
+ dependsOn(":myriad-scheduler:build")
def subdir = new File(project.buildDir, "libs")
if( !subdir.exists() ) { subdir.mkdirs() }
from '../myriad-scheduler/build/libs'
into 'libs'
}
-task copyMyriadJars(dependsOn: ":myriad-scheduler:build") {
+task copyExecutorJars(type:Copy){
dependsOn(":myriad-executor:build")
+ def subdir = new File(project.buildDir, "libs")
+ if( !subdir.exists() ) { subdir.mkdirs() }
+ from '../myriad-executor/build/libs'
+ into 'libs'
+}
+
+
+task copyMyriadJars(dependsOn: ":myriad-scheduler:build") {
dependsOn(copySchedulerJars)
+ dependsOn(copyExecutorJars)
}
task buildRMDocker(type: DockerBuildImage) {
@@ -46,5 +56,9 @@ task buildRMDocker(type: DockerBuildImage) {
url = 'unix:///var/run/docker.sock'
}
inputDir = file('.')
- tag = 'mesos/myriad'
+ if(!project.hasProperty('dockerTag')) {
+ tag = 'mesos/myriad'
+ } else {
+ tag = dockerTag
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/configure-yarn.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/configure-yarn.sh b/docker/myriad-bin/configure-yarn.sh
new file mode 100755
index 0000000..ff85ba9
--- /dev/null
+++ b/docker/myriad-bin/configure-yarn.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+: '
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+'
+##
+# YARN installer script for Apache Myriad Deployment
+##
+
+# Put in env defaults if they are missing
+export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
+
+export YARN_USER=${YARN_USER:='yarn'}
+export USER_UID=${USER_UID:='107'}
+export YARN_GROUP=${YARN_GROUP:='yarn'}
+export HADOOP_GID=${HADOOP_GID:='112'}
+export YARN_GID=${YARN_GID:='113'}
+export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
+
+# Add hduser user
+groupadd ${HADOOP_GROUP} -g ${HADOOP_GID}
+groupadd ${YARN_GROUP} -g ${YARN_GID}
+useradd ${YARN_USER} -g ${YARN_GROUP} -G ${HADOOP_GROUP} -u ${USER_UID} -s /bin/bash
+mkdir /home/${HADOOP_USER}
+chown -R ${YARN_USER}:${YARN_GROUP} /home/${YARN_USER}
+
+#set permissions
+chown -R root:root ${HADOOP_HOME}
+chmod -R g-w /usr/local/
+chown root:${YARN_GROUP} ${HADOOP_HOME}/bin/container-executor
+chmod 6050 ${HADOOP_HOME}/bin/container-executor
+
+# Init bashrc with hadoop env variables
+sh -c 'echo export JAVA_HOME=/usr >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/bin >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/sbin >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_MAPRED_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_COMMON_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_HDFS_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export YARN_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_HOME\}/lib/native >> /home/${HADOOP_USER}/.bashrc'
+sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\${HADOOP_HOME}/lib\" >> /home/${HADOOP_USER}/.bashrc'
+
+
+echo "end of configure-yarn.sh script"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/create-user.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/create-user.sh b/docker/myriad-bin/create-user.sh
deleted file mode 100755
index c6c1bf6..0000000
--- a/docker/myriad-bin/create-user.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-: '
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-'
-# Put in env defaults if they are missing
-export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
-export HADOOP_USER=${HADOOP_USER:='yarn'}
-export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
-export USER_UID=${USER_UID:='113'}
-export GROUP_UID=${GROUP_GID:='112'}
-
-# Add hduser user
-echo "Creating $HADOOP_USER user.."
-groupadd $HADOOP_GROUP -g ${GROUP_UID}
-useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash -d /home/${HADOOP_USER}
-mkdir /home/${HADOOP_USER}
-chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER}
-
-echo "end of create-user.sh script"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-bin/install-yarn.sh
----------------------------------------------------------------------
diff --git a/docker/myriad-bin/install-yarn.sh b/docker/myriad-bin/install-yarn.sh
index 41b23ed..90c2762 100755
--- a/docker/myriad-bin/install-yarn.sh
+++ b/docker/myriad-bin/install-yarn.sh
@@ -21,60 +21,28 @@ under the License.
# YARN installer script for Apache Myriad Deployment
##
HADOOP_VER="2.7.0"
-HADOOP_TARBALL_URL=http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
echo "Installing Yarn...."
-if [ ! -z "$1" ];then
- HADOOP_TARBALL_URL=$1
- echo "Deleting previous hadoop home"
- rm -rf ${HADOOP_HOME}
+if [ ! -z $1 ];then
+ HADOOP_URL=$1
+else
+ HADOOP_URL=http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz
fi
-
-# Download the tarball
-wget -O /opt/hadoop.tgz ${HADOOP_TARBALL_URL}
-HADOOP_BASENAME=`basename ${HADOOP_TARBALL_URL} .tar.gz`
-
-# Put in env defaults if they are missing
-export HADOOP_GROUP=${HADOOP_GROUP:='hadoop'}
-export HADOOP_USER=${HADOOP_USER:='yarn'}
+HADOOP_TGZ=`basename ${HADOOP_URL}`
+HADOOP_BASENAME=`basename ${HADOOP_URL} .tar.gz`
export HADOOP_HOME=${HADOOP_HOME:='/usr/local/hadoop'}
-export USER_UID=${USER_UID:='113'}
-export GROUP_UID=${GROUP_GID:='112'}
-
-# Add hduser user
-groupadd $HADOOP_GROUP -g ${GROUP_UID}
-useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash
-#mkdir /home/${HADOOP_USER}
-chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER}
# Extract Hadoop
-tar vxzf /opt/hadoop.tgz -C /tmp
-#mv /tmp/hadoop-${HADOOP_VER} ${HADOOP_HOME}
-echo "Moving /tmp/hadoop-${HADOOP_BASENAME} to ${HADOOP_HOME}"
+echo "Downloading ${HADOOP_TGZ} from ${HADOOP_URL}"
+wget ${HADOOP_URL}
+tar xzf ${HADOOP_TGZ} -C /tmp
mv /tmp/${HADOOP_BASENAME} ${HADOOP_HOME}
-ls -lath ${HADOOP_HOME}
-
-mkdir /home/$HADOOP_USER
-chown -R ${HADOOP_USER}:${HADOOP_GROUP} ${HADOOP_HOME}
-
-# Init bashrc with hadoop env variables
-sh -c 'echo export JAVA_HOME=/usr >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/bin >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export PATH=\$PATH:\${HADOOP_HOME}/sbin >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_MAPRED_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_COMMON_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_HDFS_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export YARN_HOME=\${HADOOP_HOME} >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_HOME\}/lib/native >> /home/${HADOOP_USER}/.bashrc'
-sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\${HADOOP_HOME}/lib\" >> /home/${HADOOP_USER}/.bashrc'
+#Remove tarball
+rm -f ${HADOOP_TGZ}
# Link Mesos Libraries
-touch ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
echo "export JAVA_HOME=/usr" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
echo "export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
-
# Ensure the hadoop-env is executable
chmod +x ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
-
echo "end of install-yarn.sh script"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/mapred-site.xml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/mapred-site.xml.template b/docker/myriad-etc/mapred-site.xml.template
new file mode 100644
index 0000000..b21660f
--- /dev/null
+++ b/docker/myriad-etc/mapred-site.xml.template
@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+<!--s option enables dynamic port assignment by mesos -->
+<configuration>
+<property>
+ <name>mapreduce.shuffle.port</name>
+ <value>${myriad.mapreduce.shuffle.port}</value>
+</property>
+<property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+</property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/myriad-config-default.yml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/myriad-config-default.yml.template b/docker/myriad-etc/myriad-config-default.yml.template
new file mode 100644
index 0000000..6a1e52a
--- /dev/null
+++ b/docker/myriad-etc/myriad-config-default.yml.template
@@ -0,0 +1,113 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##
+
+# Address of the mesos master - <IP:port> or ZooKeeper path
+mesosMaster: zk://CHANGEME:2181/mesos
+#Container information for the node managers
+containerInfo:
+ type: DOCKER
+ dockerInfo:
+ image: CHANGEME/myriad
+ volume:
+ -
+ containerPath: /tmp
+ hostPath: /tmp
+# Whether to check point myriad's mesos framework or not
+checkpoint: false
+# Myriad's mesos framework failover timeout in milliseconds. This tells mesos
+# to expect myriad would failover within this time interval.
+frameworkFailoverTimeout: 60000 #10 minutes
+# Myriad's mesos framework name.
+frameworkName: MyriadAlpha
+# Myriad's mesos framework role.
+#frameworkRole: myriad
+# User the Node Manager will run as (Defaults to user running the resource manager if absent, necessary for remote distribution).
+frameworkUser: yarn
+# User that gets the nodeManagerUri and sets up the directories for Node Manager, must have passwordless sudo (Necessary only for remote distribution, otherwise ignored).
+# frameworkSuperUser: root
+# Myriad's REST-ful services port mapping.
+restApiPort: 8192
+# Address of the ZK ensemble (separate by comma, if multiple zk servers are used)
+#zkServers: localhost:2181
+# ZK Session timeout
+zkTimeout: 20000
+haEnabled: True
+# The node manager profiles. The REST API to flex up expects one of the profiles defined here.
+# Admin can define custom profiles (requires restart of Resource Manager)
+nmInstances:
+ medium: 1
+profiles:
+ zero:
+ cpu: 0
+ mem: 0
+ small:
+ cpu: 1
+ mem: 1024
+ medium:
+ cpu: 2
+ mem: 2048
+ large:
+ cpu: 4
+ mem: 4096
+ extralarge:
+ cpu: 6
+ mem: 6144
+# Whether to turn on myriad's auto-rebalancer feature.
+# Currently it's work-in-progress and should be set to 'false'.
+rebalancer: false
+# Properties for the Node Manager process that's launched by myriad as a result of 'flex up' REST call.
+nodemanager:
+ jvmMaxMemoryMB: 1024 # Xmx for NM JVM process.
+ cpus: 0.2 # CPU needed by NM process.
+# cgroups: true # Whether NM should support CGroups. If set to 'true', myriad automatically
+ # configures yarn-site.xml to attach YARN's cgroups under Mesos' cgroup hierarchy.
+executor:
+ jvmMaxMemoryMB: 256 # Xmx for myriad's executor that launches Node Manager.
+ #path: hdfs://172.31.2.176:54310/dist/myriad-executor-runnable-x.x.x.jar # Path for the myriad's executor binary.
+ nodeManagerUri: hdfs://172.31.2.167:54310/dist/hadoop-2.6.0.tgz # the uri to d/l hadoop from # Path to the Hadoop tarball
+# Environment variables required to launch Node Manager process. Admin can also pass other environment variables to NodeManager.
+yarnEnvironment:
+ YARN_HOME: /usr/local/hadoop # Or /usr/local/hadoop if using MapR's Hadoop
+ #YARN_HOME: hadoop-2.6.0 # Should be relative nodeManagerUri is set
+ #YARN_NODEMANAGER_OPTS: -Dnodemanager.resource.io-spindles=4.0 # Required only if using MapR's Hadoop
+ JAVA_HOME: /usr/ # System dependent, but sometimes necessary
+services:
+ jobhistory:
+ command: $YARN_HOME/bin/mapred historyserver
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ maxInstances: 1
+ #ports:
+ # myriad.mapreduce.jobhistory.admin.address: -1
+ # myriad.mapreduce.jobhistory.address: -1
+ # myriad.mapreduce.jobhistory.webapp.address: -1
+ envSettings: -Dcluster.name.prefix=/mycluster
+ taskName: jobhistory
+ timelineserver:
+ command: $YARN_HOME/bin/yarn timelineserver
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ envSettings: -Dcluster.name.prefix=/mycluster2
+ taskName: timelineserver
+
+
+# Authentication principal for Myriad's mesos framework
+#mesosAuthenticationPrincipal: some_principal
+# Authentication secret filename for Myriad's mesos framework
+#mesosAuthenticationSecretFilename: /path/to/secret/filename
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/docker/myriad-etc/yarn-site.xml.template
----------------------------------------------------------------------
diff --git a/docker/myriad-etc/yarn-site.xml.template b/docker/myriad-etc/yarn-site.xml.template
new file mode 100644
index 0000000..c81b124
--- /dev/null
+++ b/docker/myriad-etc/yarn-site.xml.template
@@ -0,0 +1,162 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+
+
+<configuration>
+
+ <!--The hostname of the Resource Manager, this MUST be changed! -->
+ <!--If you plan to deploy via marathon, use <taskId>.marathon.mesos where taskId is the name of your marathon task-->
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>CHANGEME</value>
+ </property>
+
+ <!-- Necessary for log aggregation -->
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>Where to aggregate logs to.</description>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/var/log/hadoop-yarn/apps</value>
+ </property>
+
+ <!-- Aux Services needed for Map/Reduce and Myriad -->
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,myriad_executor</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.store.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.MyriadFileSystemRMStateStore</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
+ <value>/var/yarn/rm/system</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.recovery.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.myriad_executor.class</name>
+ <value>org.apache.myriad.executor.MyriadExecutorAuxService</value>
+ </property>
+
+ <!-- Myriad specific YARN configuration properties-->
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>${nodemanager.resource.cpu-vcores}</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>${nodemanager.resource.memory-mb}</value>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>0</value>
+ </property>
+
+ <!-- Dynamic Port Assignment enabled by Mesos -->
+ <property>
+ <name>yarn.nodemanager.address</name>
+ <value>${myriad.yarn.nodemanager.address}</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.webapp.address</name>
+ <value>${myriad.yarn.nodemanager.webapp.address}</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.webapp.https.address</name>
+ <value>${myriad.yarn.nodemanager.webapp.address}</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.localizer.address</name>
+ <value>${myriad.yarn.nodemanager.localizer.address}</value>
+ </property>
+
+ <!-- Myriad Scheduler configuration -->
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.myriad.scheduler.yarn.MyriadFairScheduler</value>
+ </property>
+
+ <!-- Cgroups configuration -->
+ <!--
+ <property>
+ <description>who will execute(launch) the containers.</description>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+ </property>
+
+ <property>
+ <description>The class which should help the LCE handle resources.</description>
+ <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler</value>
+ </property>
+
+ <property>
+ <description>The class which should help the LCE handle resources.</description>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+ <value>${yarn.nodemanager.linux-container-executor.cgroups.mount-path}</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.path</name>
+ <value>${yarn.home}/bin/container-executor</value>
+ </property>
+-->
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/build.gradle
----------------------------------------------------------------------
diff --git a/myriad-scheduler/build.gradle b/myriad-scheduler/build.gradle
index fc7516d..3e88c03 100644
--- a/myriad-scheduler/build.gradle
+++ b/myriad-scheduler/build.gradle
@@ -38,7 +38,6 @@ dependencies {
compile "org.apache.commons:commons-lang3:3.4"
compile 'com.google.inject.extensions:guice-multibindings:3.0'
testCompile "org.apache.hadoop:hadoop-yarn-server-resourcemanager:${hadoopVer}:tests"
-
}
configurations {
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
index 67c8232..716a20a 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Optional;
import com.google.common.base.Strings;
import java.util.Map;
+
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.hibernate.validator.constraints.NotEmpty;
@@ -53,8 +54,27 @@ import org.hibernate.validator.constraints.NotEmpty;
* executor:
* jvmMaxMemoryMB: 256
* path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+ * containerInfo:
+ * DockerInfo:
+ * image: mesos/myriad
+ * forcePullImage: false
+ * parameters:
+ * -
+ * key: key
+ * value: value
+ * -
+ * key: key
+ * value: value
+ * volumes:
+ * -
+ * containerPath: path
+ * hostPath: path
+ * mode: RW
+ * -
+ * containerPath: path
+ * hostPath: path
* yarnEnvironment:
- * YARN_HOME: /usr/local/hadoop
+ * YARN_HOME: /usr/local/hadoop
*/
public class MyriadConfiguration {
/**
@@ -93,6 +113,9 @@ public class MyriadConfiguration {
private Boolean checkpoint;
@JsonProperty
+ private MyriadContainerConfiguration containerInfo;
+
+ @JsonProperty
private Double frameworkFailoverTimeout;
@JsonProperty
@@ -160,7 +183,6 @@ public class MyriadConfiguration {
public MyriadConfiguration() {
}
-
public String getMesosMaster() {
return mesosMaster;
}
@@ -169,6 +191,10 @@ public class MyriadConfiguration {
return this.checkpoint != null ? checkpoint : DEFAULT_CHECKPOINT;
}
+ public Optional<MyriadContainerConfiguration> getContainerInfo() {
+ return Optional.fromNullable(containerInfo);
+ }
+
public Double getFrameworkFailoverTimeout() {
return this.frameworkFailoverTimeout != null ? this.frameworkFailoverTimeout : DEFAULT_FRAMEWORK_FAILOVER_TIMEOUT_MS;
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
new file mode 100644
index 0000000..6a6a8bc
--- /dev/null
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadContainerConfiguration.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * MyriadContainerConfiguration
+ * Provides deserialization support for containerInfo data stored in yaml config
+ */
+public class MyriadContainerConfiguration {
+ @JsonProperty
+ @NotEmpty
+ private String type;
+ @JsonProperty
+ private MyriadDockerConfiguration dockerInfo;
+ @JsonProperty
+ private List<Map<String, String>> volumes;
+
+ @JsonProperty
+ public String getType() {
+ return type;
+ }
+
+ public Optional<MyriadDockerConfiguration> getDockerInfo() {
+ return Optional.fromNullable(dockerInfo);
+ }
+
+ public Iterable<Map<String, String>> getVolumes() {
+ return volumes == null ? new ArrayList<Map<String, String>>() : volumes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
new file mode 100644
index 0000000..38a2f0f
--- /dev/null
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadDockerConfiguration.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.myriad.configuration;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.hibernate.validator.constraints.NotEmpty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * MyriadDockerConfiguration
+ * Provides deserialization support for dockerInfo data stored in yaml config
+ */
+public class MyriadDockerConfiguration {
+ @JsonProperty
+ @NotEmpty
+ String image;
+
+ @JsonProperty
+ String network;
+
+ @JsonProperty
+ Boolean privledged;
+
+ @JsonProperty
+ Boolean forcePullImage;
+
+ @JsonProperty
+ List<Map<String, String>> parameters;
+
+ public String getImage() {
+ return image;
+ }
+
+ public Boolean getForcePullImage() {
+ return forcePullImage != null ? forcePullImage : false;
+ }
+
+ public String getNetwork() {
+ return network != null ? network : "HOST";
+ }
+
+ public Boolean getPrivledged() {
+ return privledged != null ? privledged : false;
+ }
+
+ public List<Map<String, String>> getParameters() {
+ if (parameters == null) {
+ return new ArrayList<>();
+ } else {
+ return parameters;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
index 076ebbb..dd86ac0 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/ServiceTaskFactoryImpl.java
@@ -140,6 +140,9 @@ public class ServiceTaskFactoryImpl implements TaskFactory {
taskBuilder.addResources(Resource.newBuilder().setName("ports").setType(Value.Type.RANGES).setRanges(valueRanger.build()));
}
taskBuilder.setCommand(commandInfo);
+ if (cfg.getContainerInfo().isPresent()) {
+ taskBuilder.setContainer(taskUtils.getContainerInfo());
+ }
return taskBuilder.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index ecf0276..9ed54a7 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -201,10 +201,13 @@ public interface TaskFactory {
ExecutorID executorId = ExecutorID.newBuilder()
.setValue(EXECUTOR_PREFIX + frameworkId.getValue() + offer.getId().getValue() + offer.getSlaveId().getValue())
.build();
- return ExecutorInfo.newBuilder().setCommand(commandInfo).setName(EXECUTOR_NAME).setExecutorId(executorId)
- .addAllResources(taskUtils.getScalarResource(offer, "cpus", taskUtils.getExecutorCpus(), 0.0))
- .addAllResources(taskUtils.getScalarResource(offer, "mem", taskUtils.getExecutorMemory(), 0.0))
- .build();
+ ExecutorInfo.Builder executorInfo = ExecutorInfo.newBuilder().setCommand(commandInfo).setName(EXECUTOR_NAME).setExecutorId(executorId)
+ .addAllResources(taskUtils.getScalarResource(offer, "cpus", taskUtils.getExecutorCpus(), 0.0))
+ .addAllResources(taskUtils.getScalarResource(offer, "mem", taskUtils.getExecutorMemory(), 0.0));
+ if (cfg.getContainerInfo().isPresent()) {
+ executorInfo.setContainer(taskUtils.getContainerInfo());
+ }
+ return executorInfo.build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
index 845c3be..1d9c518 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
@@ -18,11 +18,15 @@
*/
package org.apache.myriad.scheduler;
+import com.google.common.base.Function;
import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
-import java.util.ArrayList;
+import java.util.*;
+import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -40,12 +44,9 @@ import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
+import com.google.common.base.Preconditions;
import org.apache.mesos.Protos;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
-import org.apache.myriad.configuration.MyriadConfiguration;
-import org.apache.myriad.configuration.MyriadExecutorConfiguration;
-import org.apache.myriad.configuration.NodeManagerConfiguration;
-import org.apache.myriad.configuration.ServiceConfiguration;
+import org.apache.myriad.configuration.*;
import org.apache.myriad.executor.MyriadExecutorDefaults;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,11 +61,19 @@ import org.xml.sax.SAXException;
*/
public class TaskUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(TaskUtils.class);
-
private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES = "yarn.nodemanager.resource.cpu-vcores";
private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB = "yarn.nodemanager.resource.memory-mb";
+ private static final String CONTAINER_PATH_KEY = "containerPath";
+ private static final String HOST_PATH_KEY = "hostPath";
+ private static final String RW_MODE = "mode";
+ private static final String CONTAINER_PORT_KEY = "containerPort";
+ private static final String HOST_PORT_KEY = "hostPort";
+ private static final String PROTOCOL_KEY = "protocol";
+ private static final String PARAMETER_KEY_KEY = "key";
+ private static final String PARAMETER_VALUE_KEY = "value";
private MyriadConfiguration cfg;
+ Random random = new Random();
@Inject
public TaskUtils(MyriadConfiguration cfg) {
@@ -208,14 +217,84 @@ public class TaskUtils {
return auxConf.getJvmMaxMemoryMB().get();
}
+ public TaskUtils() {
+ super();
+ }
+
+ public Iterable<Protos.Volume> getVolumes(Iterable<Map<String, String>> volume) {
+ return Iterables.transform(volume, new Function<Map<String, String>, Protos.Volume>() {
+ @Nullable
+ @Override
+ public Protos.Volume apply(Map<String, String> map) {
+ Preconditions.checkArgument(map.containsKey(HOST_PATH_KEY) && map.containsKey(CONTAINER_PATH_KEY));
+ Protos.Volume.Mode mode = Protos.Volume.Mode.RO;
+ if (map.containsKey(RW_MODE) && map.get(RW_MODE).toLowerCase().equals("rw")) {
+ mode = Protos.Volume.Mode.RW;
+ }
+ return Protos.Volume.newBuilder()
+ .setContainerPath(map.get(CONTAINER_PATH_KEY))
+ .setHostPath(map.get(HOST_PATH_KEY))
+ .setMode(mode)
+ .build();
+ }
+ });
+ }
+
+ public Iterable<Protos.Parameter> getParameters(Iterable<Map<String, String>> params) {
+ Preconditions.checkNotNull(params);
+ return Iterables.transform(params, new Function<Map<String, String>, Protos.Parameter>() {
+ @Override
+ public Protos.Parameter apply(Map<String, String> parameter) {
+ Preconditions.checkNotNull(parameter, "Null parameter");
+ Preconditions.checkState(parameter.containsKey(PARAMETER_KEY_KEY), "Missing key");
+ Preconditions.checkState(parameter.containsKey(PARAMETER_VALUE_KEY), "Missing value");
+ return Protos.Parameter.newBuilder()
+ .setKey(parameter.get(PARAMETER_KEY_KEY))
+ .setValue(PARAMETER_VALUE_KEY)
+ .build();
+ }
+ });
+ }
+
+ private Protos.ContainerInfo.DockerInfo getDockerInfo(MyriadDockerConfiguration dockerConfiguration) {
+ Preconditions.checkArgument(dockerConfiguration.getNetwork().equals("HOST"), "Currently only host networking supported");
+ Protos.ContainerInfo.DockerInfo.Builder dockerBuilder = Protos.ContainerInfo.DockerInfo.newBuilder()
+ .setImage(dockerConfiguration.getImage())
+ .setForcePullImage(dockerConfiguration.getForcePullImage())
+ .setNetwork(Protos.ContainerInfo.DockerInfo.Network.valueOf(dockerConfiguration.getNetwork()))
+ .setPrivileged(dockerConfiguration.getPrivledged())
+ .addAllParameters(getParameters(dockerConfiguration.getParameters()));
+ return dockerBuilder.build();
+ }
+
+ /**
+ * Builds a ContainerInfo Object
+ *
+ * @return ContainerInfo
+ */
+ public Protos.ContainerInfo getContainerInfo() {
+ Preconditions.checkArgument(cfg.getContainerInfo().isPresent(), "ContainerConfiguration doesn't exist!");
+ MyriadContainerConfiguration containerConfiguration = cfg.getContainerInfo().get();
+ Protos.ContainerInfo.Builder containerBuilder = Protos.ContainerInfo.newBuilder()
+ .setType(Protos.ContainerInfo.Type.valueOf(containerConfiguration.getType()))
+ .addAllVolumes(getVolumes(containerConfiguration.getVolumes()));
+ if (containerConfiguration.getDockerInfo().isPresent()) {
+ MyriadDockerConfiguration dockerConfiguration = containerConfiguration.getDockerInfo().get();
+ containerBuilder.setDocker(getDockerInfo(dockerConfiguration));
+ }
+ return containerBuilder.build();
+ }
+
+
/**
* Helper function that returns all scalar resources of a given name in an offer up to a given value. Attempts to
* take resource from the prescribed role first and then from the default role. The variable used indicated any
* resources previously requested. Assumes enough resources are present.
+ *
* @param offer - An offer by Mesos, assumed to have enough resources.
* @param name - The name of the SCALAR resource, i.e. cpus or mem
* @param value - The amount of SCALAR resources needed.
- * @param used - The amount of SCALAR resources already removed from this offer.
+ * @param used - The amount of SCALAR resources already removed from this offer.
* @return An Iterable containing one or two scalar resources of a given name in an offer up to a given value.
*/
public Iterable<Protos.Resource> getScalarResource(Protos.Offer offer, String name, Double value, Double used) {
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index a04b94b..14e0a00 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -19,13 +19,17 @@ package org.apache.myriad.scheduler;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import org.apache.mesos.Protos;
import org.apache.myriad.configuration.MyriadBadConfigurationException;
import org.apache.myriad.configuration.MyriadConfiguration;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -35,12 +39,20 @@ import static org.junit.Assert.fail;
public class TestTaskUtils {
static MyriadConfiguration cfg;
+ static MyriadConfiguration cfgWithRole;
+ static MyriadConfiguration cfgWithDocker;
+ static double epsilon = .0001;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
cfg = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default.yml"),
MyriadConfiguration.class);
+ cfgWithRole = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-framework-role.yml"),
+ MyriadConfiguration.class);
+ cfgWithDocker = mapper.readValue(Thread.currentThread().getContextClassLoader().getResource("myriad-config-test-default-with-docker-info.yml"),
+ MyriadConfiguration.class);
+
}
@AfterClass
@@ -74,7 +86,6 @@ public class TestTaskUtils {
Gson gson = new GsonBuilder().registerTypeAdapter(ServiceResourceProfile.class, new ServiceResourceProfile.CustomDeserializer())
.create();
-
ServiceResourceProfile parentProfile = new ServiceResourceProfile("abc", 1.0, 100.0);
String parentStr = gson.toJson(parentProfile);
@@ -98,4 +109,85 @@ public class TestTaskUtils {
new Throwable().printStackTrace();
}
+ private Protos.Offer createScalarOffer(String name, double roleVal, double defaultVal) {
+ Protos.Offer offer = Protos.Offer.newBuilder()
+ .setId(Protos.OfferID.newBuilder().setValue("offerId"))
+ .setSlaveId(Protos.SlaveID.newBuilder().setValue("slaveId"))
+ .setHostname("test.com")
+ .setFrameworkId(Protos.FrameworkID.newBuilder().setValue("frameworkId"))
+ .addResources(
+ Protos.Resource.newBuilder()
+ .setScalar(Protos.Value.Scalar.newBuilder().setValue(roleVal))
+ .setType(Protos.Value.Type.SCALAR)
+ .setName(name)
+ .setRole("test")
+ .build())
+ .addResources(
+ Protos.Resource.newBuilder()
+ .setScalar(Protos.Value.Scalar.newBuilder().setValue(defaultVal))
+ .setType(Protos.Value.Type.SCALAR)
+ .setName("cpus")
+ .build())
+ .build();
+ return offer;
+ }
+
+ private void checkResourceList(Iterable<Protos.Resource> resources, String name, Double roleVal, Double defaultVal) {
+ int i = 0;
+ Range defaultValueRange = Ranges.closed(defaultVal - epsilon, defaultVal + epsilon);
+ Range roleValueRange = Ranges.closed(roleVal - epsilon, roleVal + epsilon);
+
+ for (Protos.Resource resource: resources) {
+ if (resource.hasRole() && resource.getRole().equals("test")) {
+ double v = resource.getScalar().getValue();
+ assertTrue("Test Role has " + v + " " + name + " should have " + roleVal, roleValueRange.contains(v));
+ i++;
+ } else {
+ double v = resource.getScalar().getValue();
+ assertTrue("Default Role has " + v + " " + name + " should have " + defaultVal , defaultValueRange.contains(v));
+ i++;
+ }
+ }
+ assertTrue("There should be at most 2 resources", i <= 2);
+ }
+
+ @Test
+ public void testGetScalarResourcesWithRole() {
+ TaskUtils taskUtils = new TaskUtils(cfgWithRole);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 3.0, 2.0), "cpus", 1.0, 0.0), "cpus", 1.0, 0.0);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 1.5, 2.0), "cpus", 2.0, 1.0), "cpus", 0.5, 1.5);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 1.5, 2.0), "cpus", 1.5, 2.0), "cpus", 0.0, 1.5);
+ }
+
+ @Test
+ public void testGetScalarResources() {
+ TaskUtils taskUtils = new TaskUtils(cfg);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 0.0), "cpus", 0.0, 1.0);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 1.0, 1.0), "cpus", 0.0, 1.0);
+ checkResourceList(taskUtils.getScalarResource(createScalarOffer("cpus", 0.0, 2.0), "cpus", 0.5, 1.5), "cpus", 0.0, 0.5);
+ }
+
+ @Test
+ public void testContainerInfo() {
+ TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
+ Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
+ assertTrue("The container should have a docker", containerInfo.hasDocker());
+ assertTrue("There should be two volumes", containerInfo.getVolumesCount() == 2);
+ assertTrue("The first volume should be read only", containerInfo.getVolumes(0).getMode().equals(Protos.Volume.Mode.RO));
+ assertTrue("The first volume should be read write", containerInfo.getVolumes(1).getMode().equals(Protos.Volume.Mode.RW));
+ }
+
+ @Test public void testDockerInfo() {
+ TaskUtils taskUtils = new TaskUtils(cfgWithDocker);
+ Protos.ContainerInfo containerInfo = taskUtils.getContainerInfo();
+ assertTrue("The container should have a docker", containerInfo.hasDocker());
+ assertTrue("There should be two volumes", containerInfo.getVolumesList().size() == 2);
+ assertTrue("There should be a docker image", containerInfo.getDocker().hasImage());
+ assertTrue("The docker image should be mesos/myraid", containerInfo.getDocker().getImage().equals("mesos/myriad"));
+ assertTrue("Should be using host networking", containerInfo.getDocker().getNetwork().equals(Protos.ContainerInfo.DockerInfo.Network.HOST));
+ assertTrue("There should be two parameters", containerInfo.getDocker().getParametersList().size() == 2);
+ assertTrue("Privledged mode should be false", containerInfo.getDocker().getPrivileged() == false);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
new file mode 100644
index 0000000..221607a
--- /dev/null
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-docker-info.yml
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+mesosMaster: 10.0.2.15:5050
+checkpoint: false
+frameworkFailoverTimeout: 43200000
+frameworkName: MyriadTest
+frameworkRole: test
+frameworkUser: hduser # User the Node Manager runs as, required if nodeManagerURI set, otherwise defaults to the user
+ # running the resource manager.
+frameworkSuperUser: root # To be depricated, currently permissions need set by a superuser due to Mesos-1790. Must be
+ # root or have passwordless sudo. Required if nodeManagerURI set, ignored otherwise.
+nativeLibrary: /usr/local/lib/libmesos.so
+zkServers: localhost:2181
+zkTimeout: 20000
+restApiPort: 8192
+profiles:
+ small:
+ cpu: 1
+ mem: 1100
+ medium:
+ cpu: 2
+ mem: 2048
+ large:
+ cpu: 4
+ mem: 4096
+rebalancer: false
+nodemanager:
+ jvmMaxMemoryMB: 1024
+ cpus: 0.2
+ cgroups: false
+executor:
+ jvmMaxMemoryMB: 256
+ path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+ #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
+ nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
+ #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
+yarnEnvironment:
+ YARN_HOME: /usr/local/hadoop
+ #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
+ #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+services:
+ jobhistory:
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ ports:
+ myriad.mapreduce.jobhistory.admin.address: 0
+ myriad.mapreduce.jobhistory.address: 0
+ myriad.mapreduce.jobhistory.webapp.address: 0
+ envSettings: -Dcluster.name.prefix=/mycluster
+ taskName: jobhistory
+ timelineserver:
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ envSettings: -Dcluster.name.prefix=/mycluster2
+ taskName: timelineserver
+containerInfo:
+ type: DOCKER
+ dockerInfo:
+ image: mesos/myriad
+ parameters:
+ -
+ key: volume-driver
+ value: keywhiz
+ -
+ key: volume
+ value: all-my-secrets:/etc/secrets
+ volumes:
+ -
+ hostPath: /srv/data1/hadoop-yarn
+ containerPath: /data1/hadoop-yarn
+ mode: RO
+ -
+ hostPath: /srv/data2/hadoop-yarn
+ containerPath: /data2/hadoop-yarn
+ mode: RW
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/bcd1fc8c/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
new file mode 100644
index 0000000..ab7f137
--- /dev/null
+++ b/myriad-scheduler/src/test/resources/myriad-config-test-default-with-framework-role.yml
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+mesosMaster: 10.0.2.15:5050
+checkpoint: false
+frameworkFailoverTimeout: 43200000
+frameworkName: MyriadTest
+frameworkRole: test
+frameworkUser: hduser # User the Node Manager runs as, required if nodeManagerURI set, otherwise defaults to the user
+ # running the resource manager.
+frameworkSuperUser: root # To be depricated, currently permissions need set by a superuser due to Mesos-1790. Must be
+ # root or have passwordless sudo. Required if nodeManagerURI set, ignored otherwise.
+nativeLibrary: /usr/local/lib/libmesos.so
+zkServers: localhost:2181
+zkTimeout: 20000
+restApiPort: 8192
+profiles:
+ small:
+ cpu: 1
+ mem: 1100
+ medium:
+ cpu: 2
+ mem: 2048
+ large:
+ cpu: 4
+ mem: 4096
+rebalancer: false
+nodemanager:
+ jvmMaxMemoryMB: 1024
+ cpus: 0.2
+ cgroups: false
+executor:
+ jvmMaxMemoryMB: 256
+ path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar
+ #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
+ nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz
+ #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar
+yarnEnvironment:
+ YARN_HOME: /usr/local/hadoop
+ #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set
+ #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary
+services:
+ jobhistory:
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ ports:
+ myriad.mapreduce.jobhistory.admin.address: 0
+ myriad.mapreduce.jobhistory.address: 0
+ myriad.mapreduce.jobhistory.webapp.address: 0
+ envSettings: -Dcluster.name.prefix=/mycluster
+ taskName: jobhistory
+ timelineserver:
+ jvmMaxMemoryMB: 1024
+ cpus: 1
+ envSettings: -Dcluster.name.prefix=/mycluster2
+ taskName: timelineserver
\ No newline at end of file