You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2022/01/16 22:38:41 UTC
[couchdb] branch jenkins-dynamic-matrix updated (ceb15c1 -> fa798ff)
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a change to branch jenkins-dynamic-matrix
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard ceb15c1 Try a simpler matrix first
discard 3763885 Hard-code sm_ver for now
discard e23c6f1 Switch to branch for now
discard bc8b17a Try specifying commit hash for version
discard dc91ed2 Fix syntax error
discard 593f2aa Try dynamicMatrix community plugin
new fa798ff Dynamically generate parallel stages
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ceb15c1)
\
N -- N -- N refs/heads/jenkins-dynamic-matrix (fa798ff)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build-aux/Jenkinsfile.full | 360 ++++++++++++++++++++++++++-------------------
1 file changed, 206 insertions(+), 154 deletions(-)
[couchdb] 01/01: Dynamically generate parallel stages
Posted by ko...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch jenkins-dynamic-matrix
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit fa798ff1f5cda618bd64079e88b262fbcb8aeb42
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Sun Jan 16 17:38:18 2022 -0500
Dynamically generate parallel stages
---
build-aux/Jenkinsfile.full | 396 +++++++++++++++++++++++----------------------
1 file changed, 199 insertions(+), 197 deletions(-)
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index f1eb35e..dcdf803 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -49,18 +49,189 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
ERLANG_VERSION = '24.2'
-platform_images = [
- 'centos7': "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}",
- 'centos8': "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}",
- 'bionic': "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}",
- 'focal': "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}",
- 'stretch': "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}",
- 'buster': "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}",
- 'bullseye': "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+meta = [
+ 'centos7': [
+ name: 'CentOS 7',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'centos8': [
+ name: 'CentOS 8',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'bionic': [
+ name: 'Ubuntu 18.04',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'focal': [
+ name: 'Ubuntu 20.04',
+ spidermonkey_vsn: '68',
+ image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'stretch': [
+ name: 'Debian 9',
+ spidermonkey_vsn: '1.8.5',
+ image: "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'buster': [
+ name: 'Debian 10',
+ spidermonkey_vsn: '60',
+ image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'bullseye': [
+ name: 'Debian 11',
+ spidermonkey_vsn: '78',
+ image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+ ],
+
+ 'freebsd': [
+ name: 'FreeBSD',
+ spidermonkey_vsn: '1.8.5',
+ gnu_make: 'gmake'
+ ],
+
+ 'macos': [
+ name: 'macOS',
+ spidermonkey_vsn: '60',
+ gnu_make: 'make'
+ ]
]
gnu_make = ['freebsd': 'gmake', 'macos': 'make']
+// Credit to https://stackoverflow.com/a/69222555 for this technique.
+// We use an initial stage to dynamically generate the list of parallel stages
+// to run afterwards. This keeps our Jenkinsfile DRY and gives us more
+// flexibility and better UX than the matrix pipeline.
+def parallelStagesMap
+
+def generateNativeStage(platform) {
+ return {
+ stage(meta[platform].name) {
+ agent {
+ label platform
+ }
+
+ options {
+ skipDefaultCheckout()
+ timeout(time: 90, unit: "MINUTES")
+ }
+
+ steps {
+ // deleteDir is OK here because we're not inside of a Docker container!
+ deleteDir()
+ unstash 'tarball'
+ withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+meta[platform].gnu_make]) {
+ sh( script: unpack, label: 'Unpack tarball' )
+ dir( "${platform}/build/couchdb" ) {
+ sh 'pwd'
+ sh 'ls -l'
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh '$MAKE'
+ sh '$MAKE eunit'
+ sh '$MAKE elixir-suite'
+ sh '$MAKE exunit'
+ sh '$MAKE mango-test'
+ }
+ }
+ }
+
+ post {
+ always {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ }
+ failure {
+ dir( "${platform}/build/couchdb" ) {
+ sh 'make build-report'
+ }
+ }
+ cleanup {
+ sh 'killall -9 beam.smp || true'
+ sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
+ }
+ }
+ }
+ }
+}
+
+def generateContainerStage(platform) {
+ return {
+ stage(meta[platform].name) {
+ agent {
+ docker {
+ image meta[platform].image
+ label 'docker'
+ args "${DOCKER_ARGS}"
+ registryUrl 'https://docker.io/'
+ registryCredentialsId 'dockerhub_creds'
+ }
+ }
+
+ options {
+ skipDefaultCheckout()
+ timeout(time: 90, unit: "MINUTES")
+ }
+
+ stages {
+ stage('Build from tarball & test') {
+ steps {
+ sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
+ unstash 'tarball'
+ sh( script: unpack, label: 'Unpack tarball' )
+ dir( "${platform}/build/couchdb" ) {
+ sh 'pwd'
+ sh 'ls -l'
+ sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+ sh 'make'
+ sh 'make eunit'
+ sh 'make elixir-suite'
+ sh 'make exunit'
+ sh 'make mango-test'
+ }
+ }
+ post {
+ always {
+ junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+ }
+ failure {
+ dir( "${platform}/build/couchdb" ) {
+ sh 'make build-report'
+ }
+ }
+ }
+ }
+
+ stage('Build CouchDB packages') {
+ steps {
+ unstash 'tarball'
+ sh( script: make_packages, label: 'Build packages' )
+ sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
+ }
+ post {
+ success {
+ archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+ }
+ }
+ }
+ }
+
+ post {
+ cleanup {
+ sh 'rm -rf ${WORKSPACE}/*'
+ }
+ }
+ }
+ }
+}
+
pipeline {
// no top-level agent; agents must be declared for each stage
@@ -127,200 +298,31 @@ pipeline {
}
} // stage Build Release Tarball
- // TODO Rework once Improved Docker Pipeline Engine is released
- // https://issues.jenkins-ci.org/browse/JENKINS-47962
- // https://issues.jenkins-ci.org/browse/JENKINS-48050
-
- stage('Test and Package') {
- matrix {
- axes {
- axis {
- name 'platform'
- values 'freebsd', 'macos', 'centos7', 'centos8', 'bionic', 'focal', 'stretch', 'buster', 'bullseye'
- }
- axis {
- name 'spidermonkey'
- values '1.8.5', '60', '68', '78'
- }
- }
- excludes {
- exclude {
- axis {
- name 'platform'
- values 'centos7', 'bionic', 'stretch', 'freebsd'
- }
- axis {
- name 'spidermonkey'
- notValues '1.8.5'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'centos8', 'buster', 'macos'
- }
- axis {
- name 'spidermonkey'
- notValues '60'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'focal'
- }
- axis {
- name 'spidermonkey'
- notValues '68'
- }
- }
- exclude {
- axis {
- name 'platform'
- values 'bullseye'
+ stage('Generate Test Matrix') {
+ agent {
+ label 'master'
+ }
+ steps {
+ script {
+ parallelStagesMap = meta.collectEntries { key, platform ->
+ if (platform.image) {
+ [key: generateContainerStage(key)]
}
- axis {
- name 'spidermonkey'
- notValues '78'
+ else {
+ [key: generateNativeStage(key)]
}
}
}
+ }
+ }
- stages {
- stage('Other Build') {
- when {
- beforeAgent true
- anyOf {
- environment name: 'platform', value: 'macos'
- environment name: 'platform', value: 'freebsd'
- }
- }
-
- agent {
- label env.platform
- }
-
- options {
- skipDefaultCheckout()
- timeout(time: 90, unit: "MINUTES")
- }
-
- steps {
- // deleteDir is OK here because we're not inside of a Docker container!
- deleteDir()
- unstash 'tarball'
- withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+gnu_make[env.platform]]) {
- sh( script: unpack, label: 'Unpack tarball' )
- dir( "${platform}/build/couchdb" ) {
- sh 'pwd'
- sh 'ls -l'
- sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
- sh '$MAKE'
- sh '$MAKE eunit'
- sh '$MAKE elixir-suite'
- sh '$MAKE exunit'
- sh '$MAKE mango-test'
- }
- }
- }
-
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- failure {
- dir( "${platform}/build/couchdb" ) {
- sh 'make build-report'
- }
- }
- cleanup {
- sh 'killall -9 beam.smp || true'
- sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
- }
- } // post
-
- } // non-containerized platform
-
- stage('Container Build') {
- when {
- beforeAgent true
- not {
- anyOf {
- environment name: 'platform', value: 'macos'
- environment name: 'platform', value: 'freebsd'
- }
- }
- }
-
- agent {
- docker {
- image platform_images["${platform}"]
- label 'docker'
- args "${DOCKER_ARGS}"
- registryUrl 'https://docker.io/'
- registryCredentialsId 'dockerhub_creds'
- }
- }
-
- options {
- skipDefaultCheckout()
- timeout(time: 90, unit: "MINUTES")
- }
-
- stages {
- stage('Build from tarball & test') {
- steps {
- sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
- unstash 'tarball'
- sh( script: unpack, label: 'Unpack tarball' )
- dir( "${platform}/build/couchdb" ) {
- sh 'pwd'
- sh 'ls -l'
- sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
- sh 'make'
- sh 'make eunit'
- sh 'make elixir-suite'
- sh 'make exunit'
- sh 'make mango-test'
- }
- }
- post {
- always {
- junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
- }
- failure {
- dir( "${platform}/build/couchdb" ) {
- sh 'make build-report'
- }
- }
- }
- }
-
- stage('Build CouchDB packages') {
- steps {
- unstash 'tarball'
- sh( script: make_packages, label: 'Build packages' )
- sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
- }
- post {
- success {
- archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
- }
- }
- }
- }
-
- post {
- cleanup {
- sh 'rm -rf ${WORKSPACE}/*'
- }
- } // post
- } // containerized platform
- } // stages
-
- } // matrix
-
- } // stage 'Test and Package'
+ stage('Test and Package') {
+ steps {
+ script {
+ parallel parallelStagesMap
+ }
+ }
+ }