You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by kw...@apache.org on 2022/06/15 09:03:55 UTC
[sling-tooling-jenkins] branch feature/parallel-builds updated (a84d1ef -> 50bdb0d)
This is an automated email from the ASF dual-hosted git repository.
kwin pushed a change to branch feature/parallel-builds
in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
discard a84d1ef SLING-9948 parallelize steps
new 50bdb0d SLING-9948 parallelize steps
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a84d1ef)
\
N -- N -- N refs/heads/feature/parallel-builds (50bdb0d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
vars/slingOsgiBundleBuild.groovy | 29 ++++++++++++++++-------------
1 file changed, 16 insertions(+), 13 deletions(-)
[sling-tooling-jenkins] 01/01: SLING-9948 parallelize steps
Posted by kw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kwin pushed a commit to branch feature/parallel-builds
in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
commit 50bdb0db9ba5d4477714fda7b3aea582c59dfee5
Author: Konrad Windszus <kw...@apache.org>
AuthorDate: Wed Jun 15 09:47:47 2022 +0200
SLING-9948 parallelize steps
reuse build artifacts for SonarCloud analysis
deploy only if every other previous stage was successfully executed
---
.../apache/sling/jenkins/SlingJenkinsHelper.groovy | 51 +---
vars/slingOsgiBundleBuild.groovy | 268 +++++++++++++++------
2 files changed, 201 insertions(+), 118 deletions(-)
diff --git a/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy b/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
index da46c65..fc94635 100644
--- a/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
+++ b/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
@@ -43,57 +43,10 @@ def static jsonArrayToCsv(net.sf.json.JSONArray items) {
}
-def runWithErrorHandling(Closure build) {
-
- def jobConfig = [
- jdks: [8],
- upstreamProjects: [],
- archivePatterns: [],
- mavenGoal: '',
- additionalMavenParams: '',
- rebuildFrequency: '@weekly',
- enabled: true,
- emailRecipients: [],
- sonarQubeEnabled: true,
- sonarQubeUseAdditionalMavenParams: true,
- sonarQubeAdditionalParams: ''
- ]
+def runWithErrorHandling(def jobConfig, Closure build) {
try {
- timeout(time:30, unit: 'MINUTES', activity: true) {
-
- stage('Init') {
- checkout scm
- sh "git clean -fdx"
- def url = sh(returnStdout: true, script: 'git config remote.origin.url').trim()
- jobConfig.repoName = url.substring(url.lastIndexOf('/') + 1).replace('.git', '');
- if ( fileExists('.sling-module.json') ) {
- overrides = readJSON file: '.sling-module.json'
- echo "Jenkins overrides: ${overrides.jenkins}"
- overrides.jenkins.each { key,value ->
- jobConfig[key] = value;
- }
- }
- echo "Final job config: ${jobConfig}"
- }
-
- stage('Configure Job') {
- def upstreamProjectsCsv = jobConfig.upstreamProjects ?
- jsonArrayToCsv(jobConfig.upstreamProjects) : ''
- def jobTriggers = []
- if ( env.BRANCH_NAME == 'master' )
- jobTriggers.add(cron(jobConfig.rebuildFrequency))
- if ( upstreamProjectsCsv )
- jobTriggers.add(upstream(upstreamProjects: upstreamProjectsCsv, threshold: hudson.model.Result.SUCCESS))
-
- properties([
- pipelineTriggers(jobTriggers),
- buildDiscarder(logRotator(numToKeepStr: '10'))
- ])
- }
-
- build.call(jobConfig)
- }
+ build.call()
// exception handling copied from https://github.com/apache/maven-jenkins-lib/blob/d6c76aaea9df19ad88439eba4f9d1ad6c9e272bd/vars/asfMavenTlpPlgnBuild.groovy
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
// this ambiguous condition means a user probably aborted
diff --git a/vars/slingOsgiBundleBuild.groovy b/vars/slingOsgiBundleBuild.groovy
index 45ede41..1e7cc11 100644
--- a/vars/slingOsgiBundleBuild.groovy
+++ b/vars/slingOsgiBundleBuild.groovy
@@ -9,70 +9,104 @@ def call(Map params = [:]) {
githubCredentialsId: 'sling-github-token'
]
+ def helper = new SlingJenkinsHelper()
+ def jobConfig = [
+ jdks: [8],
+ upstreamProjects: [],
+ archivePatterns: [],
+ mavenGoal: '',
+ additionalMavenParams: '',
+ rebuildFrequency: '@weekly',
+ enabled: true,
+ emailRecipients: [],
+ sonarQubeEnabled: true,
+ sonarQubeUseAdditionalMavenParams: true,
+ sonarQubeAdditionalParams: ''
+ ]
+
node(globalConfig.mainNodeLabel) {
+ timeout(time:5, unit: 'MINUTES') {
+ stage('Init') {
+ checkout scm
+ sh "git clean -fdx"
+ def url = sh(returnStdout: true, script: 'git config remote.origin.url').trim()
+ jobConfig.repoName = url.substring(url.lastIndexOf('/') + 1).replace('.git', '');
+ if ( fileExists('.sling-module.json') ) {
+ overrides = readJSON file: '.sling-module.json'
+ echo "Jenkins overrides: ${overrides.jenkins}"
+ overrides.jenkins.each { key,value ->
+ jobConfig[key] = value;
+ }
+ }
+ echo "Final job config: ${jobConfig}"
+ }
+ }
+ }
- def helper = new SlingJenkinsHelper()
-
- helper.runWithErrorHandling({ jobConfig ->
- if ( jobConfig.enabled ) {
-
- // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
- // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
- def isReferenceStage = true
-
- jobConfig.jdks.each { jdkVersion ->
- stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
- stageDefinition.call()
- isReferenceStage = false
- currentBuild.result = "SUCCESS"
+ if ( isOnMainBranch() ) {
+ node(globalConfig.mainNodeLabel) {
+ timeout(time:30, unit: 'MINUTES', activity: true) {
+ stage('Configure Job') {
+ def upstreamProjectsCsv = jobConfig.upstreamProjects ?
+ jsonArrayToCsv(jobConfig.upstreamProjects) : ''
+ def jobTriggers = []
+ if ( env.BRANCH_NAME == 'master' )
+ jobTriggers.add(cron(jobConfig.rebuildFrequency))
+ if ( upstreamProjectsCsv )
+ jobTriggers.add(upstream(upstreamProjects: upstreamProjectsCsv, threshold: hudson.model.Result.SUCCESS))
+
+ properties([
+ pipelineTriggers(jobTriggers),
+ buildDiscarder(logRotator(numToKeepStr: '10'))
+ ])
}
+ }
+ }
+ }
- // this might fail if there are no jdks defined, but that's always an error
- // also, we don't activate any Maven publisher since we don't want this part of the
- // build tracked, but using withMaven(...) allows us to easily reuse the same
- // Maven and JDK versions
- def additionalMavenParams = additionalMavenParams(jobConfig)
- def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
+ if ( jobConfig.enabled ) {
+ helper.runWithErrorHandling(jobConfig, {
+ // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
+ // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
+ def isReferenceStage = true
- if ( jobConfig.sonarQubeEnabled ) {
- stage('SonarCloud') {
- // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
- def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
- if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
- sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
- }
- // Params are different if it's a PR or if it's not
- // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
- if ( isPrBuild ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
- } else if ( env.BRANCH_NAME != "master" ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
- }
- static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
- // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
- withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
- // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
- withMaven(maven: globalConfig.mvnVersion,
- jdk: jenkinsJdkLabel(11, globalConfig),
- publisherStrategy: 'EXPLICIT') {
- try {
- sh "mvn -U clean verify ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams} -Pci"
- } catch ( Exception e ) {
- // TODO - we should check the actual failure cause here, but see
- // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
- echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
- currentBuild.result = 'UNSTABLE'
- }
- }
+ // contains the label as key and a closure to execute as value
+ def stepsMap = [:]
+ // parallel execution of all build jobs (reference potentially including SonarQube)
+ jobConfig.jdks.each { jdkVersion ->
+ stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
+ stepsMap["$jdkVersion"] = stageDefinition.call()
+ isReferenceStage = false
+ currentBuild.result = "SUCCESS"
+ }
+
+ // do a quick sanity check first without tests if multiple parallel builds are required
+ if ( stepsMap.size > 1 ) {
+ node(globalConfig.mainNodeLabel) {
+ stage("Sanity Check") {
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(8, globalConfig), // TODO: use Java version from first Job Config
+ publisherStrategy: 'EXPLICIT') {
+ sh "mvn clean compile ${additionalMavenParams(jobConfig)}"
}
}
- } else {
- echo "SonarQube execution is disabled"
}
- } else {
- echo "Job is disabled, not building"
+ }
+
+ // execute the actual Maven builds
+ parallel stepsMap
+
+ // last stage is deploy
+ if ( shouldDeploy('deploy') ) {
+ node(globalConfig.mainNodeLabel) {
+ stage("Deploy to Nexus") {
+ deployToNexus()
+ }
+ }
}
})
+ } else {
+ echo "Job is disabled, not building"
}
}
@@ -97,18 +131,21 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
// do not deploy artifacts built from PRs or feature branches
// also do not deploy non-SNAPSHOT versions
- if ( goal == "deploy" ) {
- def notMaster = env.BRANCH_NAME != "master"
- def mavenPom = readMavenPom()
- def mavenVersion = mavenPom.version ?: mavenPom.parent.version
- def isSnapshot = mavenVersion.endsWith('-SNAPSHOT')
- if ( notMaster || !isSnapshot ) {
- goal = "verify"
- echo "Maven goal set to ${goal} since branch is not master ( ${env.BRANCH_NAME} ) or version is not snapshot ( ${mavenVersion} )"
- }
+ if ( goal == "deploy" && ! shouldDeploy(goal) ) {
+ goal = "verify"
+ echo "Maven goal set to ${goal} since branch is not master ( ${env.BRANCH_NAME} ) or version is not snapshot"
}
def invocation = {
+ if ( isReferenceStage && shouldDeploy(goal) ) {
+ String localRepoPath = "${env.WORKSPACE}/local-snapshots-dir"
+ // Make sure the directory is wiped.
+ dir(localRepoPath) {
+ deleteDir()
+ }
+ // main build with IT for properly calculating coverage
+ additionalMavenParams = "${additionalMavenParams} -DaltDeploymentRepository=snapshot-repo::default::file:${localRepoPath}"
+ }
withMaven(maven: globalConfig.mvnVersion, jdk: jenkinsJdkLabel,
options: [
artifactsPublisher(disabled: !isReferenceStage),
@@ -122,21 +159,114 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
if ( isReferenceStage && jobConfig.archivePatterns ) {
archiveArtifacts(artifacts: SlingJenkinsHelper.jsonArrayToCsv(jobConfig.archivePatterns), allowEmptyArchive: true)
}
+ if ( isReferenceStage && shouldDeploy(goal) ) {
+ // Stash the build results so we can deploy them on another node
+ stash name: 'local-snapshots-dir', includes: 'local-snapshots-dir/**'
+ }
}
def branchConfig = jobConfig?.branches?."$env.BRANCH_NAME" ?: [:]
- if ( branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel )
- invocation = wrapInNode(invocation,branchConfig.nodeLabel)
-
return {
- stage("Build (Java ${jdkVersion}, ${goal})") {
- invocation.call()
+ wrapInNode(branchConfig.nodeLabel ?: globalConfig.mainNodeLabel, {
+ timeout(time: 30, unit: 'MINUTES') {
+ stage("Build (Java ${jdkVersion}, ${goal})") {
+ invocation.call()
+ }
+ }
+ if ( isReferenceStage ) {
+ // SonarQube (must be wrapped in same node)
+ if ( jobConfig.sonarQubeEnabled ) {
+ stage('Analyse with SonarCloud') {
+ timeout(time: 30, unit: 'MINUTES') {
+ analyseWithSonarCloud(globalConfig, jobConfig)
+ }
+ }
+ }
+ }
+ })
+ }
+}
+
+def analyseWithSonarCloud(def globalConfig, def jobConfig) {
+ // this might fail if there are no jdks defined, but that's always an error
+ // also, we don't activate any Maven publisher since we don't want this part of the
+ // build tracked, but using withMaven(...) allows us to easily reuse the same
+ // Maven and JDK versions
+ def additionalMavenParams = additionalMavenParams(jobConfig)
+ def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
+
+ // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
+ def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
+ if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
+ sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
+ }
+ // Params are different if it's a PR or if it's not
+ // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
+ if ( isPrBuild ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
+ } else if ( env.BRANCH_NAME != "master" ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
+ }
+ static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
+ // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
+ withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
+ // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(11, globalConfig),
+ publisherStrategy: 'EXPLICIT') {
+ try {
+ sh "mvn ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams}"
+ } catch ( Exception e ) {
+ // TODO - we should check the actual failure cause here, but see
+ // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
+ echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
+ currentBuild.result = 'UNSTABLE'
+ }
}
}
}
-def wrapInNode(Closure invocation, def nodeLabel) {
+def deployToNexus() {
+ node('nexus-deploy') {
+ timeout(60) {
+ echo "Running on node ${env.NODE_NAME}"
+ // first clear workspace
+ deleteDir()
+ // Nexus deployment needs pom.xml
+ checkout scm
+ // Unstash the previously stashed build results.
+ unstash name: 'local-snapshots-dir'
+ // https://www.mojohaus.org/wagon-maven-plugin/merge-maven-repos-mojo.html
+ String mavenArguments = "${wagonPluginGav}:merge-maven-repos -Dwagon.target=https://repository.apache.org/content/repositories/snapshots -Dwagon.targetId=apache.snapshots.https -Dwagon.source=file:${env.WORKSPACE}/local-snapshots-dir"
+ pipelineSupport.executeMaven(this, mavenArguments, false)
+ }
+ }
+}
+
+boolean shouldDeploy(String goal) {
+ // check branch name
+ if ( !isOnMainBranch() ) {
+ return false
+ }
+ if (goal != 'deploy') {
+ return false
+ }
+ // check version
+ def mavenPom = readMavenPom()
+ def mavenVersion = mavenPom.version ?: mavenPom.parent.version
+ def isSnapshot = mavenVersion.endsWith('-SNAPSHOT')
+ if ( !isSnapshot ) {
+ return false
+ }
+ return true
+}
+
+boolean isOnMainBranch() {
+ return env.BRANCH_NAME == "master"
+}
+
+def wrapInNode(def nodeLabel, Closure invocation) {
return {
node(nodeLabel) {
checkout scm