You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2022/01/16 22:38:42 UTC
[couchdb] 01/01: Dynamically generate parallel stages
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch jenkins-dynamic-matrix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fa798ff1f5cda618bd64079e88b262fbcb8aeb42
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Sun Jan 16 17:38:18 2022 -0500
Dynamically generate parallel stages
---
build-aux/Jenkinsfile.full | 396 +++++++++++++++++++++++----------------------
1 file changed, 199 insertions(+), 197 deletions(-)
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index f1eb35e..dcdf803 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -49,18 +49,189 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
ERLANG_VERSION = '24.2'
-platform_images = [
- 'centos7': "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}",
- 'centos8': "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}",
- 'bionic': "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}",
- 'focal': "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}",
- 'stretch': "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}",
- 'buster': "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}",
- 'bullseye': "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+meta = [
+ 'centos7': [
+ name: 'CentOS 7',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'centos8': [
+ name: 'CentOS 8',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'bionic': [
+ name: 'Ubuntu 18.04',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'focal': [
+ name: 'Ubuntu 20.04',
+ spidermonkey_vsn: '68',
+ image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'stretch': [
+ name: 'Debian 9',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'buster': [
+ name: 'Debian 10',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'bullseye': [
+ name: 'Debian 11',
+ spidermonkey_vsn: '78',
+ image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'freebsd': [
+ name: 'FreeBSD',
+ spidermonkey_vsn: '1.8.5',
+ gnu_make: 'gmake'
+ ],
+
+ 'macos': [
+ name: 'macOS',
+ spidermonkey_vsn: '60',
+ gnu_make: 'make'
+ ]
]
gnu_make = ['freebsd': 'gmake', 'macos': 'make']
+// Credit to https://stackoverflow.com/a/69222555 for this technique.
+// We use an initial stage to dynamically generate the list of parallel stages
+// to run afterwards. This keeps our Jenkinsfile DRY and gives us more
+// flexibility and better UX than the matrix pipeline.
+def parallelStagesMap
+
+def generateNativeStage(platform) {
+ return {
+ stage(meta[platform].name) {
+ agent {
+ label platform
+ }
+
+ options {
+ skipDefaultCheckout()
+ timeout(time: 90, unit: "MINUTES")
+ }
+
+ steps {
+ // deleteDir is OK here because we're not inside of a Docker container!
+ deleteDir()
+ unstash 'tarball'
+ withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+meta[platform].gnu_make]) {
+ sh( script: unpack, label: 'Unpack tarball' )
+ dir( "${platform}/build/couchdb" ) {
+ sh 'pwd'
+ sh 'ls -l'
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh '$MAKE'
+ sh '$MAKE eunit'
+ sh '$MAKE elixir-suite'
+ sh '$MAKE exunit'
+ sh '$MAKE mango-test'
+ }
+ }
+ }
+
+ post {
+ always {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ }
+ failure {
+ dir( "${platform}/build/couchdb" ) {
+ sh 'make build-report'
+ }
+ }
+ cleanup {
+ sh 'killall -9 beam.smp || true'
+ sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
+ }
+ }
+ }
+ }
+}
+
+def generateContainerStage(platform) {
+ return {
+ stage(meta[platform].name) {
+ agent {
+ docker {
+ image meta[platform].image
+ label 'docker'
+ args "${DOCKER_ARGS}"
+ registryUrl 'https://docker.io/'
+ registryCredentialsId 'dockerhub_creds'
+ }
+ }
+
+ options {
+ skipDefaultCheckout()
+ timeout(time: 90, unit: "MINUTES")
+ }
+
+ stages {
+ stage('Build from tarball & test') {
+ steps {
+ sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
+ unstash 'tarball'
+ sh( script: unpack, label: 'Unpack tarball' )
+ dir( "${platform}/build/couchdb" ) {
+ sh 'pwd'
+ sh 'ls -l'
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh 'make'
+ sh 'make eunit'
+ sh 'make elixir-suite'
+ sh 'make exunit'
+ sh 'make mango-test'
+ }
+ }
+ post {
+ always {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ }
+ failure {
+ dir( "${platform}/build/couchdb" ) {
+ sh 'make build-report'
+ }
+ }
+ }
+ }
+
+ stage('Build CouchDB packages') {
+ steps {
+ unstash 'tarball'
+ sh( script: make_packages, label: 'Build packages' )
+ sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
+ }
+ post {
+ success {
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ }
+ }
+ }
+ }
+
+ post {
+ cleanup {
+ sh 'rm -rf ${WORKSPACE}/*'
+ }
+ }
+ }
+ }
+}
+
pipeline {
// no top-level agent; agents must be declared for each stage
@@ -127,200 +298,31 @@ pipeline {
}
} // stage Build Release Tarball
- // TODO Rework once Improved Docker Pipeline Engine is released
- // https://issues.jenkins-ci.org/browse/JENKINS-47962
- // https://issues.jenkins-ci.org/browse/JENKINS-48050
-
- stage('Test and Package') {
- matrix {
- axes {
- axis {
- name 'platform'
- values 'freebsd', 'macos', 'centos7', 'centos8', 'bionic', 'focal', 'stretch', 'buster', 'bullseye'
- }
- axis {
- name 'spidermonkey'
- values '1.8.5', '60', '68', '78'
- }
- }
- excludes {
- exclude {
- axis {
- name 'platform'
- values 'centos7', 'bionic', 'stretch', 'freebsd'
- }
- axis {
- name 'spidermonkey'
- notValues '1.8.5'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'centos8', 'buster', 'macos'
- }
- axis {
- name 'spidermonkey'
- notValues '60'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'focal'
- }
- axis {
- name 'spidermonkey'
- notValues '68'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'bullseye'
+ stage('Generate Test Matrix') {
+ agent {
+ label 'master'
+ }
+ steps {
+ script {
+ parallelStagesMap = meta.collectEntries { key, platform ->
+ if (platform.image) {
+ [key: generateContainerStage(key)]
}
- axis {
- name 'spidermonkey'
- notValues '78'
+ else {
+ [key: generateNativeStage(key)]
}
}
}
+ }
+ }
- stages {
- stage('Other Build') {
- when {
- beforeAgent true
- anyOf {
- environment name: 'platform', value: 'macos'
- environment name: 'platform', value: 'freebsd'
- }
- }
-
- agent {
- label env.platform
- }
-
- options {
- skipDefaultCheckout()
- timeout(time: 90, unit: "MINUTES")
- }
-
- steps {
- // deleteDir is OK here because we're not inside of a Docker container!
- deleteDir()
- unstash 'tarball'
- withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+gnu_make[env.platform]]) {
- sh( script: unpack, label: 'Unpack tarball' )
- dir( "${platform}/build/couchdb" ) {
- sh 'pwd'
- sh 'ls -l'
- sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
- sh '$MAKE'
- sh '$MAKE eunit'
- sh '$MAKE elixir-suite'
- sh '$MAKE exunit'
- sh '$MAKE mango-test'
- }
- }
- }
-
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- failure {
- dir( "${platform}/build/couchdb" ) {
- sh 'make build-report'
- }
- }
- cleanup {
- sh 'killall -9 beam.smp || true'
- sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
- }
- } // post
-
- } // non-containerized platform
-
- stage('Container Build') {
- when {
- beforeAgent true
- not {
- anyOf {
- environment name: 'platform', value: 'macos'
- environment name: 'platform', value: 'freebsd'
- }
- }
- }
-
- agent {
- docker {
- image platform_images["${platform}"]
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
-
- options {
- skipDefaultCheckout()
- timeout(time: 90, unit: "MINUTES")
- }
-
- stages {
- stage('Build from tarball & test') {
- steps {
- sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
- unstash 'tarball'
- sh( script: unpack, label: 'Unpack tarball' )
- dir( "${platform}/build/couchdb" ) {
- sh 'pwd'
- sh 'ls -l'
- sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
- sh 'make'
- sh 'make eunit'
- sh 'make elixir-suite'
- sh 'make exunit'
- sh 'make mango-test'
- }
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- failure {
- dir( "${platform}/build/couchdb" ) {
- sh 'make build-report'
- }
- }
- }
- }
-
- stage('Build CouchDB packages') {
- steps {
- unstash 'tarball'
- sh( script: make_packages, label: 'Build packages' )
- sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- }
-
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // containerized platform
- } // stages
-
- } // matrix
-
- } // stage 'Test and Package'
+ stage('Test and Package') {
+ steps {
+ script {
+ parallel parallelStagesMap
+ }
+ }
+ }