You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2014/07/01 21:51:51 UTC
git commit: Add a script that allows for easy selective rebuilding of
components in the vagrant environment.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 7955b63c2 -> d7b8f76b4
Add a script that allows for easy selective rebuilding of components in the vagrant environment.
Reviewed at https://reviews.apache.org/r/22948/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/d7b8f76b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/d7b8f76b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/d7b8f76b
Branch: refs/heads/master
Commit: d7b8f76b47e84a145ae32d91171e3274adb6f03f
Parents: 7955b63
Author: Bill Farner <wf...@apache.org>
Authored: Tue Jul 1 12:36:57 2014 -0700
Committer: Bill Farner <wf...@apache.org>
Committed: Tue Jul 1 12:36:57 2014 -0700
----------------------------------------------------------------------
docs/vagrant.md | 52 +++++++++---
examples/vagrant/aurorabuild.sh | 111 +++++++++++++++++++++++++
examples/vagrant/provision-dev-cluster.sh | 76 +++++------------
3 files changed, 175 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d7b8f76b/docs/vagrant.md
----------------------------------------------------------------------
diff --git a/docs/vagrant.md b/docs/vagrant.md
index 8c67672..b66ff67 100644
--- a/docs/vagrant.md
+++ b/docs/vagrant.md
@@ -1,17 +1,49 @@
-Aurora includes a `Vagrantfile` that defines a full Mesos cluster running Aurora. You can use it to
-explore Aurora's various components. To get started, install
-[VirtualBox](https://www.virtualbox.org/) and [Vagrant](http://www.vagrantup.com/),
-then run `vagrant up` somewhere in the repository source tree to create a team of VMs. This may take some time initially as it builds all
-the components involved in running an aurora cluster.
+Getting Started
+===============
+To replicate a real cluster environment as closely as possible, we use
+[Vagrant](http://www.vagrantup.com/) to launch a complete Aurora cluster in a virtual machine.
-The scheduler is listening on http://192.168.33.7:8081/scheduler
-The observer is listening on http://192.168.33.7:1338
-The master is listening on http://192.168.33.7:5050
+Prerequisites
+-------------
+ * [VirtualBox](https://www.virtualbox.org/)
+ * [Vagrant](http://www.vagrantup.com/)
+ * A clone of the Aurora repository, or source distribution.
+
+You can start a local cluster by running:
+
+ vagrant up
+
+Once started, several services should be running:
+
+ * scheduler is listening on http://192.168.33.7:8081
+ * observer is listening on http://192.168.33.7:1338
+ * master is listening on http://192.168.33.7:5050
+ * slave is listening on http://192.168.33.7:5051
+
+You can SSH into the machine with `vagrant ssh` and execute aurora client commands using the
+`aurora` command. A pre-installed `clusters.json` file refers to your local cluster as
+`devcluster`, which you will use in client commands.
+
+Deleting your local cluster
+===========================
+Once you are finished with your local cluster, or if you would otherwise like to start from scratch,
+you can use the command `vagrant destroy` to turn off and delete the virtual file system.
+
+
+Rebuilding components
+=====================
+If you are changing Aurora code and would like to rebuild a component, you can use the `aurorabuild`
+command on your vagrant machine to build and restart a component. This is considerably faster than
+destroying and rebuilding your VM.
+
+`aurorabuild` accepts a list of components to build and update. You may invoke the command with
+no arguments to get a list of supported components.
+
+ vagrant ssh -c 'sudo aurorabuild client'
-Once everything is up, you can `vagrant ssh devcluster` and execute aurora client commands using the `aurora` client.
Troubleshooting
----------------
+===============
Most of the vagrant related problems can be fixed by the following steps:
* Destroying the vagrant environment with `vagrant destroy`
* Killing any orphaned VMs (see AURORA-499) with `virtualbox` UI or `VBoxManage` command line tool
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d7b8f76b/examples/vagrant/aurorabuild.sh
----------------------------------------------------------------------
diff --git a/examples/vagrant/aurorabuild.sh b/examples/vagrant/aurorabuild.sh
new file mode 100755
index 0000000..9f604d3
--- /dev/null
+++ b/examples/vagrant/aurorabuild.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# A script to update sources within the vagrant environment an rebuild/install aurora components.
+# Usage: aurorabuild [COMPONENT]...
+# where COMPONENT is a name for an aurora component that makes up part of the infrastructure.
+# Run with zero arguments for a full list of components that may be built.
+
+set -o nounset
+
+REPO_DIR=/home/vagrant/aurora
+DIST_DIR=$REPO_DIR/dist
+AURORA_HOME=/usr/local/aurora
+
+function upstart_update {
+ # Stop and start is necessary to update a the configuration of
+ # an upstart job. We'll rarely change the configuration, but
+ # it's probably better to do this upfront and avoid surprises/confusion.
+ stop $1
+ start $1
+}
+
+function build_client {
+ ./pants src/main/python/apache/aurora/client/bin:aurora_client
+ ln -sf $DIST_DIR/aurora_client.pex /usr/local/bin/aurora
+}
+
+function build_client2 {
+ ./pants src/main/python/apache/aurora/client/cli:aurora2
+ ln -sf $DIST_DIR/aurora2.pex /usr/local/bin/aurora2
+}
+
+function build_admin_client {
+ ./pants src/main/python/apache/aurora/client/bin:aurora_admin
+ ln -sf $DIST_DIR/aurora_admin.pex /usr/local/bin/aurora_admin
+}
+
+function build_scheduler {
+ ./gradlew installApp
+
+ export LD_LIBRARY_PATH=/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server
+ mkdir -p $AURORA_HOME/scheduler
+ if sudo mesos-log initialize --path="$AURORA_HOME/scheduler/db"
+ then
+ echo "Replicated log initialized."
+ else
+ echo "Replicated log initialization failed with code $? (likely already initialized)."
+ fi
+ unset LD_LIBRARY_PATH
+ upstart_update aurora-scheduler
+}
+
+function build_executor {
+ ./pants src/main/python/apache/aurora/executor/bin:gc_executor
+ ./pants src/main/python/apache/aurora/executor/bin:thermos_executor
+ ./pants src/main/python/apache/aurora/executor/bin:thermos_runner
+
+ # Package runner within executor.
+ python <<EOF
+import contextlib
+import zipfile
+with contextlib.closing(zipfile.ZipFile('dist/thermos_executor.pex', 'a')) as zf:
+ zf.writestr('apache/aurora/executor/resources/__init__.py', '')
+ zf.write('dist/thermos_runner.pex', 'apache/aurora/executor/resources/thermos_runner.pex')
+EOF
+}
+
+function build_observer {
+ ./pants src/main/python/apache/thermos/observer/bin:thermos_observer
+ upstart_update aurora-thermos-observer
+}
+
+function print_components {
+ echo 'Please select from: client, client2, admin_client, scheduler, executor, observer.'
+}
+
+if [ "$#" -eq 0 ]
+then
+ echo 'Must specify at least one component to build'
+ print_components
+ exit 1
+fi
+
+for component in "$@"
+do
+ type "build_$component" >/dev/null && continue
+ echo "Component $component is unrecognized."
+ print_components
+ exit 1
+done
+
+cd $REPO_DIR
+update-sources
+for component in "$@"
+do
+ build_$component
+done
+
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d7b8f76b/examples/vagrant/provision-dev-cluster.sh
----------------------------------------------------------------------
diff --git a/examples/vagrant/provision-dev-cluster.sh b/examples/vagrant/provision-dev-cluster.sh
index f6459a1..c05adcd 100755
--- a/examples/vagrant/provision-dev-cluster.sh
+++ b/examples/vagrant/provision-dev-cluster.sh
@@ -30,49 +30,29 @@ update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
# that want to advertise the hostname to the user, or other components.
hostname 192.168.33.7
-function build_all() {
- echo Copying aurora source code
- rsync -urzvh /vagrant/ aurora --filter=':- .gitignore' --exclude=.git
-
+function prepare_extras() {
pushd aurora
- # fetch the mesos egg, needed to build python components
+ # Fetch the mesos egg, needed to build python components.
mkdir -p third_party
pushd third_party
wget -c http://downloads.mesosphere.io/master/ubuntu/12.04/mesos_0.18.0_amd64.egg \
-O mesos-0.18.0-py2.7-linux-x86_64.egg
popd
- # install thrift, needed for code generation in the scheduler build
+ # Install thrift, needed for code generation in the scheduler build.
# TODO(wfarner): Move deb file out of jfarrell's individual hosting.
thrift_deb=thrift-compiler_0.9.1_amd64.deb
wget -c http://people.apache.org/~jfarrell/thrift/0.9.1/contrib/deb/ubuntu/12.04/$thrift_deb
dpkg --install $thrift_deb
- # build scheduler
- ./gradlew installApp
-
- # build clients
- ./pants src/main/python/apache/aurora/client/bin:aurora_admin
- ./pants src/main/python/apache/aurora/client/bin:aurora_client
- ./pants src/main/python/apache/aurora/client/cli:aurora2
-
- # build executors/observers
- ./pants src/main/python/apache/aurora/executor/bin:gc_executor
- ./pants src/main/python/apache/aurora/executor/bin:thermos_executor
- ./pants src/main/python/apache/aurora/executor/bin:thermos_runner
- ./pants src/main/python/apache/thermos/observer/bin:thermos_observer
-
- # package runner w/in executor
- python <<EOF
-import contextlib
-import zipfile
-with contextlib.closing(zipfile.ZipFile('dist/thermos_executor.pex', 'a')) as zf:
- zf.writestr('apache/aurora/executor/resources/__init__.py', '')
- zf.write('dist/thermos_runner.pex', 'apache/aurora/executor/resources/thermos_runner.pex')
-EOF
+ # Include build script in default PATH.
+ ln -sf /home/vagrant/aurora/examples/vagrant/aurorabuild.sh /usr/local/bin/aurorabuild
popd
sudo chown -R vagrant:vagrant aurora
+
+ # Install the upstart configurations.
+ cp /vagrant/examples/vagrant/upstart/*.conf /etc/init
}
function install_mesos {
@@ -80,26 +60,7 @@ function install_mesos {
dpkg --install mesos_0.18.0_amd64.deb
}
-function install_aurora {
- # The bulk of the 'install' was done by the build, the result of which we access
- # through /home/vagrant.
- DIST_DIR=/home/vagrant/aurora/dist
- AURORA_HOME=/usr/local/aurora
-
- export LD_LIBRARY_PATH=/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/server
- mkdir -p $AURORA_HOME/scheduler
- if mesos-log initialize --path="$AURORA_HOME/scheduler/db"; then
- echo "Replicated log initialized."
- else
- echo "Replicated log initialization failed with code $? (likely already initialized)."
- fi
- unset LD_LIBRARY_PATH
-
- # Ensure clients are in the default PATH.
- ln -s $DIST_DIR/aurora_client.pex /usr/local/bin/aurora
- ln -s $DIST_DIR/aurora2.pex /usr/local/bin/aurora2
- ln -s $DIST_DIR/aurora_admin.pex /usr/local/bin/aurora_admin
-
+function install_cluster_config {
mkdir -p /etc/aurora
cat > /etc/aurora/clusters.json <<EOF
[{
@@ -114,16 +75,23 @@ EOF
}
function start_services {
- cp /vagrant/examples/vagrant/upstart/*.conf /etc/init
-
start zookeeper
start mesos-master
start mesos-slave
- start aurora-thermos-observer
- start aurora-scheduler
}
-build_all
+function prepare_sources {
+ cat > /usr/local/bin/update-sources <<EOF
+#!/bin/bash
+rsync -urzvh /vagrant/ /home/vagrant/aurora --filter=':- /vagrant/.gitignore' --exclude=.git
+EOF
+ chmod +x /usr/local/bin/update-sources
+ update-sources
+}
+
+prepare_sources
install_mesos
-install_aurora
+prepare_extras
+install_cluster_config
start_services
+aurorabuild client client2 admin_client executor observer scheduler