You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by se...@apache.org on 2017/02/08 07:56:37 UTC
aurora-packaging git commit: Adding support for Ubuntu Xenial packages
Repository: aurora-packaging
Updated Branches:
refs/heads/master 712ca6158 -> 2b19410bf
Adding support for Ubuntu Xenial packages
Added builder and test environment for Xenial as well as updated instructions
on how to test it. Added distribution to release-candidate script.
Bugs closed: AURORA-1872
Reviewed at https://reviews.apache.org/r/52437/
Project: http://git-wip-us.apache.org/repos/asf/aurora-packaging/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora-packaging/commit/2b19410b
Tree: http://git-wip-us.apache.org/repos/asf/aurora-packaging/tree/2b19410b
Diff: http://git-wip-us.apache.org/repos/asf/aurora-packaging/diff/2b19410b
Branch: refs/heads/master
Commit: 2b19410bf014d6d756772e846a97e4da61f51a92
Parents: 712ca61
Author: Renan DelValle <rd...@binghamton.edu>
Authored: Wed Feb 8 08:56:28 2017 +0100
Committer: Stephan Erb <se...@apache.org>
Committed: Wed Feb 8 08:56:28 2017 +0100
----------------------------------------------------------------------
build-support/release/release-candidate | 1 +
builder/deb/debian-jessie/Dockerfile | 1 +
builder/deb/debian-jessie/build.sh | 9 +++
builder/deb/ubuntu-trusty/Dockerfile | 1 +
builder/deb/ubuntu-trusty/build.sh | 9 +++
builder/deb/ubuntu-xenial/Dockerfile | 55 +++++++++++++++++++
builder/deb/ubuntu-xenial/build.sh | 51 +++++++++++++++++
specs/debian/aurora-executor.thermos.default | 3 +
specs/debian/aurora-executor.thermos.service | 13 +++--
specs/debian/aurora-pants.ini | 3 -
specs/debian/aurora-scheduler.service | 6 +-
specs/debian/aurora-scheduler.startup.sh | 40 ++++++++++++++
specs/debian/rules | 9 ++-
test/deb/ubuntu-xenial/README.md | 67 +++++++++++++++++++++++
test/deb/ubuntu-xenial/Vagrantfile | 14 +++++
test/deb/ubuntu-xenial/provision.sh | 17 ++++++
16 files changed, 284 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/build-support/release/release-candidate
----------------------------------------------------------------------
diff --git a/build-support/release/release-candidate b/build-support/release/release-candidate
index b3ebe91..c5aad69 100755
--- a/build-support/release/release-candidate
+++ b/build-support/release/release-candidate
@@ -20,6 +20,7 @@ cd "${ROOT_DIR}"
declare -A DIST_DIRS=(
["ubuntu-trusty"]="${ROOT_DIR}/artifacts/aurora-ubuntu-trusty/dist"
+ ["ubuntu-xenial"]="${ROOT_DIR}/artifacts/aurora-ubuntu-xenial/dist"
["debian-jessie"]="${ROOT_DIR}/artifacts/aurora-debian-jessie/dist"
["centos-7"]="${ROOT_DIR}/artifacts/aurora-centos-7/dist/rpmbuild/RPMS/x86_64"
)
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/debian-jessie/Dockerfile
----------------------------------------------------------------------
diff --git a/builder/deb/debian-jessie/Dockerfile b/builder/deb/debian-jessie/Dockerfile
index 54cb2e0..077d2b6 100644
--- a/builder/deb/debian-jessie/Dockerfile
+++ b/builder/deb/debian-jessie/Dockerfile
@@ -28,6 +28,7 @@ RUN echo \
RUN apt-get update && apt-get -y install \
bison \
debhelper \
+ dh-systemd \
devscripts \
dpkg-dev \
curl \
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/debian-jessie/build.sh
----------------------------------------------------------------------
diff --git a/builder/deb/debian-jessie/build.sh b/builder/deb/debian-jessie/build.sh
index e0aeaf5..04f4cb1 100755
--- a/builder/deb/debian-jessie/build.sh
+++ b/builder/deb/debian-jessie/build.sh
@@ -22,6 +22,15 @@ tar --strip-components 1 -C . -xf /src.tar.gz
cp -R /specs/debian .
+DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]')
+THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"
+THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/"
+
+# Place the link to the correct python egg into aurora-pants.ini
+echo "[python-repos]" >> ./debian/aurora-pants.ini
+echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini
+
export DEBFULLNAME='Apache Aurora'
export DEBEMAIL='dev@aurora.apache.org'
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-trusty/Dockerfile
----------------------------------------------------------------------
diff --git a/builder/deb/ubuntu-trusty/Dockerfile b/builder/deb/ubuntu-trusty/Dockerfile
index 802049b..7712549 100644
--- a/builder/deb/ubuntu-trusty/Dockerfile
+++ b/builder/deb/ubuntu-trusty/Dockerfile
@@ -19,6 +19,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get -y install \
bison \
debhelper \
+ dh-systemd \
devscripts \
dpkg-dev \
curl \
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-trusty/build.sh
----------------------------------------------------------------------
diff --git a/builder/deb/ubuntu-trusty/build.sh b/builder/deb/ubuntu-trusty/build.sh
index e0aeaf5..04f4cb1 100755
--- a/builder/deb/ubuntu-trusty/build.sh
+++ b/builder/deb/ubuntu-trusty/build.sh
@@ -22,6 +22,15 @@ tar --strip-components 1 -C . -xf /src.tar.gz
cp -R /specs/debian .
+DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]')
+THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"
+THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/"
+
+# Place the link to the correct python egg into aurora-pants.ini
+echo "[python-repos]" >> ./debian/aurora-pants.ini
+echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini
+
export DEBFULLNAME='Apache Aurora'
export DEBEMAIL='dev@aurora.apache.org'
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-xenial/Dockerfile
----------------------------------------------------------------------
diff --git a/builder/deb/ubuntu-xenial/Dockerfile b/builder/deb/ubuntu-xenial/Dockerfile
new file mode 100644
index 0000000..a32f4bc
--- /dev/null
+++ b/builder/deb/ubuntu-xenial/Dockerfile
@@ -0,0 +1,55 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM ubuntu:xenial
+
+WORKDIR /aurora
+ENV HOME /aurora
+ENV DEBIAN_FRONTEND noninteractive
+
+# Using gcc-4.8 to ensure ABI for thermos in containers running older distros
+RUN apt-get update && apt-get -y install \
+ bison \
+ debhelper \
+ devscripts \
+ dh-systemd \
+ dpkg-dev \
+ curl \
+ git \
+ gcc-4.8 \
+ libapr1-dev \
+ libcurl4-openssl-dev \
+ libsvn-dev \
+ libffi-dev \
+ python-all-dev \
+ software-properties-common \
+ libkrb5-dev
+
+RUN apt-get -y install openjdk-8-jdk \
+ && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+
+RUN curl -sSL http://apache.org/dist/thrift/KEYS | gpg --import - && \
+ gpg --export --armor 66B778F9 | apt-key add - && \
+ echo 'deb http://www.apache.org/dist/thrift/debian 0.9.1 main' > /etc/apt/sources.list.d/thrift.list && \
+ apt-get update && \
+ apt-get -y install thrift-compiler=0.9.1
+
+# Install gradle.
+RUN git clone --depth 1 https://github.com/benley/gradle-packaging \
+ && cd gradle-packaging \
+ && apt-get -y install ruby ruby-dev unzip wget \
+ && gem install fpm && ./gradle-mkdeb.sh 3.1 \
+ && apt-get -y install gdebi-core \
+ && gdebi --non-interactive gradle-3.1_3.1-2_all.deb \
+ && cd .. && rm -rf gradle-packaging
+
+ADD build.sh /build.sh
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-xenial/build.sh
----------------------------------------------------------------------
diff --git a/builder/deb/ubuntu-xenial/build.sh b/builder/deb/ubuntu-xenial/build.sh
new file mode 100755
index 0000000..90a9414
--- /dev/null
+++ b/builder/deb/ubuntu-xenial/build.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+mkdir /scratch
+cd /scratch
+
+tar --strip-components 1 -C . -xf /src.tar.gz
+
+cp -R /specs/debian .
+
+# Xenial tries to convert init and upstart scripts before using systemd units.
+# Avoid conflict by not including them for now.
+rm ./debian/*.upstart ./debian/*.init
+
+DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]')
+THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"
+THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/"
+
+# Place the link to the correct python egg into aurora-pants.ini
+echo "[python-repos]" >> ./debian/aurora-pants.ini
+echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini
+
+export DEBFULLNAME='Apache Aurora'
+export DEBEMAIL='dev@aurora.apache.org'
+
+dch \
+ --newversion $AURORA_VERSION \
+ --package apache-aurora \
+ --urgency medium \
+ "Apache Aurora package builder <de...@aurora.apache.org> $(date -R)"
+dch --release ''
+
+dpkg-buildpackage -uc -b -tc
+
+mkdir /dist
+mv ../*.deb /dist
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-executor.thermos.default
----------------------------------------------------------------------
diff --git a/specs/debian/aurora-executor.thermos.default b/specs/debian/aurora-executor.thermos.default
index 82ab0c8..716f08e 100644
--- a/specs/debian/aurora-executor.thermos.default
+++ b/specs/debian/aurora-executor.thermos.default
@@ -16,3 +16,6 @@
# Listen port for thermos_observer
OBSERVER_PORT=1338
+
+MESOS_ROOT=/var/lib/mesos
+EXTRA_THERMOS_OBSERVER_ARGS=""
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-executor.thermos.service
----------------------------------------------------------------------
diff --git a/specs/debian/aurora-executor.thermos.service b/specs/debian/aurora-executor.thermos.service
index da35116..32fc004 100644
--- a/specs/debian/aurora-executor.thermos.service
+++ b/specs/debian/aurora-executor.thermos.service
@@ -11,16 +11,17 @@
# limitations under the License.
[Unit]
-Description=Thermos Observer
+Description=Aurora Thermos Observer
After=network.target
Wants=network.target
[Service]
-Type=forking
-IgnoreSIGPIPE=no
-GuessMainPID=no
-ExecStart=/etc/init.d/thermos start
-ExecStop=/etc/init.d/thermos stop
+EnvironmentFile=/etc/default/thermos
+ExecStart=/usr/sbin/thermos_observer \
+ --port=${OBSERVER_PORT} \
+ --mesos-root=${MESOS_ROOT} \
+ --log_to_disk=NONE \
+ --log_to_stderr=google:INFO
User=root
Group=root
Restart=always
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-pants.ini
----------------------------------------------------------------------
diff --git a/specs/debian/aurora-pants.ini b/specs/debian/aurora-pants.ini
index e4f2c54..b2c622d 100644
--- a/specs/debian/aurora-pants.ini
+++ b/specs/debian/aurora-pants.ini
@@ -1,5 +1,2 @@
[GLOBAL]
print_exception_stacktrace: True
-
-[python-repos]
-repos: ['third_party/', 'https://svn.apache.org/repos/asf/aurora/3rdparty/ubuntu/trusty64/python/']
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-scheduler.service
----------------------------------------------------------------------
diff --git a/specs/debian/aurora-scheduler.service b/specs/debian/aurora-scheduler.service
index ae33d26..b81cb63 100644
--- a/specs/debian/aurora-scheduler.service
+++ b/specs/debian/aurora-scheduler.service
@@ -16,11 +16,7 @@ After=network.target
Wants=network.target
[Service]
-Type=forking
-IgnoreSIGPIPE=no
-GuessMainPID=no
-ExecStart=/etc/init.d/aurora-scheduler start
-ExecStop=/etc/init.d/aurora-scheduler stop
+ExecStart=/usr/bin/aurora-scheduler-startup
User=aurora
Group=aurora
Restart=always
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-scheduler.startup.sh
----------------------------------------------------------------------
diff --git a/specs/debian/aurora-scheduler.startup.sh b/specs/debian/aurora-scheduler.startup.sh
new file mode 100644
index 0000000..3f97a73
--- /dev/null
+++ b/specs/debian/aurora-scheduler.startup.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Starts up an Aurora scheduler process.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /etc/default/aurora-scheduler
+
+# Environment variables control the behavior of the Mesos scheduler driver (libmesos).
+export GLOG_v LIBPROCESS_PORT LIBPROCESS_IP
+export JAVA_OPTS="${JAVA_OPTS[*]}"
+
+# Preferences Java 1.8 over any other Java version.
+export PATH=/usr/lib/jvm/java-1.8.0/bin:${PATH}
+
+exec /usr/sbin/aurora-scheduler \
+ -cluster_name="$CLUSTER_NAME" \
+ -http_port="$HTTP_PORT" \
+ -native_log_quorum_size="$QUORUM_SIZE" \
+ -zk_endpoints="$ZK_ENDPOINTS" \
+ -mesos_master_address="$MESOS_MASTER" \
+ -serverset_path="$ZK_SERVERSET_PATH" \
+ -native_log_zk_group_path="$ZK_LOGDB_PATH" \
+ -native_log_file_path="$LOGDB_FILE_PATH" \
+ -backup_dir="$BACKUP_DIR" \
+ -thermos_executor_path="$THERMOS_EXECUTOR_PATH" \
+ -thermos_executor_resources="$THERMOS_EXECUTOR_RESOURCES" \
+ -thermos_executor_flags="$THERMOS_EXECUTOR_FLAGS" \
+ -allowed_container_types="$ALLOWED_CONTAINER_TYPES" \
+ $EXTRA_SCHEDULER_ARGS
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/rules
----------------------------------------------------------------------
diff --git a/specs/debian/rules b/specs/debian/rules
index 4effc3e..8e338b8 100755
--- a/specs/debian/rules
+++ b/specs/debian/rules
@@ -16,8 +16,10 @@
# Uncomment for debugging.
#DH_VERBOSE = 1
+TMP = $(CURDIR)/debian
+
%:
- dh $@ --exclude=aurora-scheduler.bat
+ dh $@ --exclude=aurora-scheduler.bat --with=systemd
gradle = $(shell which gradle)
pants = $(CURDIR)/pants
@@ -42,3 +44,8 @@ __pants_build:
override_dh_installinit:
dh_installinit --name=thermos
dh_installinit
+
+override_dh_auto_install:
+ dh_auto_install
+ # Install start up script for aurora-scheduler
+ install -D -m 0755 $(TMP)/aurora-scheduler.startup.sh $(TMP)/aurora-scheduler/usr/bin/aurora-scheduler-startup
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/README.md
----------------------------------------------------------------------
diff --git a/test/deb/ubuntu-xenial/README.md b/test/deb/ubuntu-xenial/README.md
new file mode 100644
index 0000000..a5284ec
--- /dev/null
+++ b/test/deb/ubuntu-xenial/README.md
@@ -0,0 +1,67 @@
+# Installing Aurora
+
+## Install packages
+
+Two methods are described, one for installing locally built packages, the other
+for installing released packages or release candidate packages.
+
+### Locally built
+
+ # Install vagrant scp
+ vagrant plugin install vagrant-scp
+
+ # Scp over the newly built packages
+ for deb in ../../../artifacts/aurora-ubuntu-xenial/dist/*.deb; do
+ vagrant scp $deb :$(basename $deb);
+ done
+
+ # Install each deb
+ vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338
+ sudo dpkg -i aurora*.deb thermos*.deb
+
+### Released
+
+ vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338
+ version=0.15.0
+ pkg_root="https://apache.bintray.com/aurora/ubuntu-xenial/"
+ for deb in \
+ aurora-scheduler_${version}_amd64.deb \
+ aurora-executor_${version}_amd64.deb \
+ aurora-tools_${version}_amd64.deb; do
+ wget $pkg_root/$deb
+ sudo dpkg -i $deb
+ done
+
+## Initialize and start
+
+The replicated log has to be initialized manually:
+
+ # Stop services first
+ sudo systemctl stop aurora-scheduler
+ sudo systemctl stop thermos
+
+ sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db
+ sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db
+ sudo systemctl start aurora-scheduler
+ sudo systemctl start thermos
+
+To make the Thermos observer work, you will have to follow the instructions of our
+[Install Guide](https://github.com/apache/aurora/blob/master/docs/operations/installation.md#configuration).
+
+## Create a job
+
+```
+echo "
+task = SequentialTask(
+ processes = [Process(name = 'hello', cmdline = 'echo hello')],
+ resources = Resources(cpu = 0.5, ram = 128*MB, disk = 128*MB))
+jobs = [Service(
+ task = task, cluster = 'example', role = 'vagrant', environment = 'prod', name = 'hello')]" > hello_world.aurora
+
+aurora job create example/vagrant/prod/hello hello_world.aurora
+```
+## Troubleshooting
+
+* Mesos: `/var/log/mesos`
+* Aurora scheduler: `cat /var/log/syslog | grep aurora-scheduler` and `sudo journalctl -u aurora-scheduler`
+* Aurora observer: `sudo journalctl -u thermos`
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/Vagrantfile
----------------------------------------------------------------------
diff --git a/test/deb/ubuntu-xenial/Vagrantfile b/test/deb/ubuntu-xenial/Vagrantfile
new file mode 100644
index 0000000..3ce6209
--- /dev/null
+++ b/test/deb/ubuntu-xenial/Vagrantfile
@@ -0,0 +1,14 @@
+Vagrant.require_version ">= 1.5.0"
+
+Vagrant.configure(2) do |config|
+ config.vm.define "aurora_xenial"
+ config.vm.box = "bento/ubuntu-16.04"
+
+ config.vm.hostname = "localhost"
+
+ config.vm.provider :virtualbox do |vb|
+ vb.customize ["modifyvm", :id, "--memory", "4096"]
+ vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+ end
+ config.vm.provision "shell", path: "provision.sh"
+end
http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/provision.sh
----------------------------------------------------------------------
diff --git a/test/deb/ubuntu-xenial/provision.sh b/test/deb/ubuntu-xenial/provision.sh
new file mode 100644
index 0000000..eda79f6
--- /dev/null
+++ b/test/deb/ubuntu-xenial/provision.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+apt-get update
+
+apt-get install -y openjdk-8-jre zookeeperd
+
+update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+
+# Mesos dependencies
+apt-get -y install libcurl3 libevent-dev libsvn1 libcurl4-nss-dev
+
+package=mesos_1.0.0-2.0.86.ubuntu1604_amd64.deb
+wget -c http://repos.mesosphere.com/ubuntu/pool/main/m/mesos/$package
+dpkg -i $package
+
+systemctl start mesos-master
+systemctl start mesos-slave