You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wo...@apache.org on 2017/07/20 01:25:42 UTC
[couchdb] 01/01: Build packages and repos
This is an automated email from the ASF dual-hosted git repository.
wohali pushed a commit to branch jenkins-packages-3
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f22c32c2794ef344c1eba8d301ed764123e502a7
Author: Joan Touzet <jo...@atypical.net>
AuthorDate: Wed Jul 19 18:48:57 2017 -0400
Build packages and repos
---
Jenkinsfile | 328 +++++++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 239 insertions(+), 89 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6fbb7e7..38397a3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,48 +26,64 @@ pipeline {
stage('Build') {
agent {
docker {
- /* This image has the oldest Erlang we support, 16B03 */
+ label 'ubuntu'
+ // This image has the oldest Erlang we support, 16B03
image 'couchdbdev/ubuntu-14.04-erlang-default'
- /* We need the jenkins user mapped inside of the image */
- args '-v /etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+ // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+ // We need the jenkins user mapped inside of the image
+ // npm config cache below is required because /home/jenkins doesn't
+ // ACTUALLY exist in the image
+ args '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0'
}
}
steps {
timeout(time: 15, unit: "MINUTES") {
- /* npm config cache below is required because /home/jenkins doesn't
- ACTUALLY exist in the image */
- /* sh 'git clone --depth 10 https://github.com/apache/couchdb .' */
sh '''
- export npm_config_cache=$(mktemp -d)
- ./configure --with-curl
- make dist
+ set
+ rm -rf apache-couchdb-*
+ ./configure --with-curl
+ make dist
+ chown -R jenkins:jenkins * || true
'''
+ }
+ }
+ post {
+ success {
stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true
deleteDir()
}
+ failure {
+ deleteDir()
+ }
}
}
- /* TODO rework this once JENKINS-41334 is released
- https://issues.jenkins-ci.org/browse/JENKINS-41334 */
- /* The builddir stuff is to prevent all 10 builds from live syncing
- their build results to each other during the build. Moving the
- build outside of the workdir should speed up the build process too,
- though it does mean we pollute /tmp whenever a build fails. */
+ // TODO rework this once JENKINS-41334 is released
+ // https://issues.jenkins-ci.org/browse/JENKINS-41334
+
+ // The builddir stuff is to prevent all the builds from live syncing
+ // their build results to each other during the build, which ACTUALLY
+ // HAPPENS. Ugh.
+
+ // we run all package builds as root inside the Docker container
+
+ // Build packages on supported platforms using esl's erlang
stage('Test') {
steps {
parallel(centos6erlang183: {
node(label: 'ubuntu') {
- timeout(time: 45, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 60, unit: "MINUTES") {
sh 'docker pull couchdbdev/centos-6-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/centos-6-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+ withDockerContainer(image: 'couchdbdev/centos-6-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ umask 0
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -75,22 +91,40 @@ pipeline {
./configure --with-curl
make all
make check || (build-aux/logfile-uploader.py && false)
+
+ # Build CouchDB packages
+ cd $builddir
+ git clone https://github.com/apache/couchdb-pkg
+ mkdir couchdb
+ cp $cwd/apache-couchdb-*.tar.gz couchdb
+ tar -xf $cwd/apache-couchdb-*.tar.gz -C couchdb
+ cd couchdb-pkg
+ platform=centos6
+ make $platform PLATFORM=$platform
+
+ # Cleanup & save for posterity
+ rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
+ mv ~/rpmbuild/RPMS/x86_64/*rpm $cwd/pkgs/$platform || true
+ chown -R jenkins:jenkins $cwd/*
'''
} // withDocker
} // timeout
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ deleteDir()
} // node
},
centos7erlangdefault: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 45, unit: "MINUTES") {
sh 'docker pull couchdbdev/centos-7-erlang-default'
- withDockerContainer(image: 'couchdbdev/centos-7-erlang-default', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+ withDockerContainer(image: 'couchdbdev/centos-7-erlang-default', args: '-e LD_LIBRARY_PATH=/usr/local/bin -e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -101,19 +135,22 @@ pipeline {
'''
} // withDocker
} // timeout
+ deleteDir()
} // node
},
centos7erlang183: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 60, unit: "MINUTES") {
sh 'docker pull couchdbdev/centos-7-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/centos-7-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+ withDockerContainer(image: 'couchdbdev/centos-7-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ umask 0
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -121,22 +158,38 @@ pipeline {
./configure --with-curl
make all
make check || (build-aux/logfile-uploader.py && false)
+
+ # Build CouchDB packages
+ cd $builddir
+ git clone https://github.com/apache/couchdb-pkg
+ mkdir couchdb
+ cp $cwd/apache-couchdb-*.tar.gz couchdb
+ tar -xf $cwd/apache-couchdb-*.tar.gz -C couchdb
+ cd couchdb-pkg
+ platform=centos7
+ make $platform PLATFORM=$platform
+
+ # Cleanup & save for posterity
+ rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
+ mv ~/rpmbuild/RPMS/x86_64/*rpm $cwd/pkgs/$platform || true
+ chown -R jenkins:jenkins $cwd/*
'''
} // withDocker
} // timeout
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ deleteDir()
} // node
},
ubuntu1204erlang183: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 45, unit: "MINUTES") {
sh 'docker pull couchdbdev/ubuntu-12.04-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/ubuntu-12.04-erlang-18.3', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/ubuntu-12.04-erlang-18.3', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -147,19 +200,19 @@ pipeline {
'''
} // withDocker
} // timeout
+ deleteDir()
} // node
},
ubuntu1404erlangdefault: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 45, unit: "MINUTES") {
sh 'docker pull couchdbdev/ubuntu-14.04-erlang-default'
- withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-default', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -170,19 +223,22 @@ pipeline {
'''
} // withDocker
} // timeout
+ deleteDir()
} // node
},
ubuntu1404erlang183: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 60, unit: "MINUTES") {
sh 'docker pull couchdbdev/ubuntu-14.04-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-18.3', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ umask 0
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -190,22 +246,39 @@ pipeline {
./configure --with-curl
make all
make check || (build-aux/logfile-uploader.py && false)
+
+ # Build CouchDB packages
+ cd $builddir
+ git clone https://github.com/apache/couchdb-pkg
+ mkdir couchdb
+ cp $cwd/apache-couchdb-*.tar.gz couchdb
+ tar -xf $cwd/apache-couchdb-*.tar.gz -C couchdb
+ cd couchdb-pkg
+ sudo apt-get install -y libmozjs185-dev
+ platform=$(lsb_release -cs)
+ make $platform PLATFORM=$platform
+
+ # Cleanup & save for posterity
+ rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
+ mv ../couchdb/*deb $cwd/pkgs/$platform || true
+ chown -R jenkins:jenkins $cwd/*
'''
} // withDocker
} // timeout
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ deleteDir()
} // node
},
ubuntu1604erlangdefault: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 45, unit: "MINUTES") {
sh 'docker pull couchdbdev/ubuntu-16.04-erlang-default'
- withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-default', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -216,19 +289,22 @@ pipeline {
'''
} // withDocker
} // timeout
+ deleteDir()
} // node
},
ubuntu1604erlang183: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 60, unit: "MINUTES") {
sh 'docker pull couchdbdev/ubuntu-16.04-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-18.3', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ umask 0
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -236,22 +312,39 @@ pipeline {
./configure --with-curl
make all
make check || (build-aux/logfile-uploader.py && false)
+
+ # Build CouchDB packages
+ cd $builddir
+ git clone https://github.com/apache/couchdb-pkg
+ mkdir couchdb
+ cp $cwd/apache-couchdb-*.tar.gz couchdb
+ tar -xf $cwd/apache-couchdb-*.tar.gz -C couchdb
+ cd couchdb-pkg
+ sudo apt-get install -y libmozjs185-dev
+ platform=$(lsb_release -cs)
+ make $platform PLATFORM=$platform
+
+ # Cleanup & save for posterity
+ rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
+ mv ../couchdb/*deb $cwd/pkgs/$platform || true
+ chown -R jenkins:jenkins $cwd/*
'''
} // withDocker
} // timeout
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ deleteDir()
} // node
},
debian8erlangdefault: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 45, unit: "MINUTES") {
sh 'docker pull couchdbdev/debian-8-erlang-default'
- withDockerContainer(image: 'couchdbdev/debian-8-erlang-default', args: '--user 0:0') {
+ withDockerContainer(image: 'couchdbdev/debian-8-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -262,19 +355,22 @@ pipeline {
'''
} // withDocker
} // timeout
+ deleteDir()
} // node
},
debian8erlang183: {
node(label: 'ubuntu') {
- timeout(time: 30, unit: "MINUTES") {
- sh 'rm *.tar.gz || true'
- unstash 'tarball'
+ timeout(time: 60, unit: "MINUTES") {
sh 'docker pull couchdbdev/debian-8-erlang-18.3'
- withDockerContainer(image: 'couchdbdev/debian-8-erlang-18.3', args: '--user 0:0') {
+ // must run as root because of sudo step below
+ withDockerContainer(image: 'couchdbdev/debian-8-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
+ sh 'rm -f apache-couchdb-*.tar.gz'
+ unstash 'tarball'
sh '''
cwd=$(pwd)
- rm -rf /tmp/couchjslogs
- mkdir -p /tmp/couchjslogs
+ mkdir -p $COUCHDB_IO_LOG_DIR
+
+ # Build CouchDB from tarball
builddir=$(mktemp -d)
cd $builddir
tar -xf $cwd/apache-couchdb-*.tar.gz
@@ -282,9 +378,27 @@ pipeline {
./configure --with-curl
make all
make check || (build-aux/logfile-uploader.py && false)
+
+ # Build CouchDB packages
+ cd $builddir
+ git clone https://github.com/apache/couchdb-pkg
+ mkdir couchdb
+ cp $cwd/apache-couchdb-*.tar.gz couchdb
+ tar -xf $cwd/apache-couchdb-*.tar.gz -C couchdb
+ cd couchdb-pkg
+ sudo apt-get install -y libmozjs185-dev
+ platform=$(lsb_release -cs)
+ make $platform PLATFORM=$platform
+
+ # Cleanup & save for posterity
+ rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
+ mv ../couchdb/*deb $cwd/pkgs/$platform || true
+ chown -R jenkins:jenkins $cwd/*
'''
} // withDocker
} // timeout
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ deleteDir()
} // node
}
) // parallel
@@ -293,22 +407,58 @@ pipeline {
stage('Publish') {
when {
- branch '*(master|2.0.x|2.1.x)'
+ expression { return env.BRANCH_NAME ==~ /master|2.0.x|2.1.x|jenkins-.*/ }
}
- agent any
- steps {
- /* Push it somewhere useful other than Jenkins, maybe? */
- /* echo 'Publishing tarball...'
- unstash 'tarball' */
- echo 'Triggering Debian .deb builds...'
- echo 'Triggering Ubuntu .deb builds...'
- echo 'Triggering Ubuntu snap builds...'
- echo 'Triggering CentOS .rpm builds...'
- echo 'Cleaning workspace...'
- sh 'rm -rf * .[a-zA-Z]*'
+ agent {
+ docker {
+ // This image has the deb AND rpm repo tools installed in it
+ image 'couchdbdev/debian-8-base'
+ // We need the jenkins user mapped inside of the image
+ // We must run as root because we have to create ~/.ssh for rsync
+ args '-v /etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+ label 'ubuntu'
+ }
}
- }
- }
+ steps {
+ withCredentials([file(credentialsId: 'jenkins-key', variable: 'KEY')]) {
+ sh 'rm -rf pkgs *.tar.gz'
+ unarchive mapping: ['pkgs/' : '.']
+ echo 'Building Debian repo...'
+ sh '''
+ git clone https://github.com/apache/couchdb-pkg
+ reprepro -b couchdb-pkg/repo includedeb jessie pkgs/jessie/*deb
+ reprepro -b couchdb-pkg/repo includedeb trusty pkgs/trusty/*deb
+ reprepro -b couchdb-pkg/repo includedeb xenial pkgs/xenial/*deb
+ '''
+ echo 'Building CentOS repos...'
+ sh '''
+ cd pkgs/centos6 && createrepo --database .
+ cd ../centos7 && rm -f js* && createrepo --database .
+ '''
+ echo 'rsyncing repos to couchdb-vm2...'
+ sh '''
+ mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el6 $BRANCH_NAME/el7
+ mv couchdb-pkg/repo/pool $BRANCH_NAME/debian
+ mv couchdb-pkg/repo/dists $BRANCH_NAME/debian
+ mv pkgs/centos6/* $BRANCH_NAME/el6
+ mv pkgs/centos7/* $BRANCH_NAME/el7
+ rsync -avz --delete -e "ssh -o StrictHostKeyChecking=no -i $KEY" $BRANCH_NAME jenkins@couchdb-vm2.apache.org:/var/www/html
+ '''
+ // cronjob on couchdb-vm2 cleans up old tarballs (keeps latest 10)
+ echo 'rsyncing source tarball to couchdb-vm2...'
+ unstash 'tarball'
+ sh '''
+ rm -rf $BRANCH_NAME
+ mkdir -p $BRANCH_NAME/source
+ mv apache-couchdb-*.tar.gz $BRANCH_NAME/source
+ rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" $BRANCH_NAME jenkins@couchdb-vm2.apache.org:/var/www/html
+ rm -rf $BRANCH_NAME couchdb-pkg *.tar.gz
+ '''
+ deleteDir()
+ } // withCredentials
+ } // steps
+ } // stage
+ } // stages
post {
success {
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.