You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by kw...@apache.org on 2022/06/15 08:34:17 UTC
[sling-tooling-jenkins] branch feature/parallel-builds updated (1f3e478 -> 5fec7fa)
This is an automated email from the ASF dual-hosted git repository.
kwin pushed a change to branch feature/parallel-builds
in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
discard 1f3e478 SLING-9948 parallelize steps
new 5fec7fa SLING-9948 parallelize steps
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1f3e478)
\
N -- N -- N refs/heads/feature/parallel-builds (5fec7fa)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
vars/slingOsgiBundleBuild.groovy | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
[sling-tooling-jenkins] 01/01: SLING-9948 parallelize steps
Posted by kw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kwin pushed a commit to branch feature/parallel-builds
in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
commit 5fec7fa50c53162ae8c9ca182c9c908b9513e50b
Author: Konrad Windszus <kw...@apache.org>
AuthorDate: Wed Jun 15 09:47:47 2022 +0200
SLING-9948 parallelize steps
reuse build artifacts for SonarCloud analysis
deploy only if every other previous stage was successfully executed
---
.../apache/sling/jenkins/SlingJenkinsHelper.groovy | 51 +----
vars/slingOsgiBundleBuild.groovy | 247 ++++++++++++++++-----
2 files changed, 188 insertions(+), 110 deletions(-)
diff --git a/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy b/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
index da46c65..fc94635 100644
--- a/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
+++ b/src/org/apache/sling/jenkins/SlingJenkinsHelper.groovy
@@ -43,57 +43,10 @@ def static jsonArrayToCsv(net.sf.json.JSONArray items) {
}
-def runWithErrorHandling(Closure build) {
-
- def jobConfig = [
- jdks: [8],
- upstreamProjects: [],
- archivePatterns: [],
- mavenGoal: '',
- additionalMavenParams: '',
- rebuildFrequency: '@weekly',
- enabled: true,
- emailRecipients: [],
- sonarQubeEnabled: true,
- sonarQubeUseAdditionalMavenParams: true,
- sonarQubeAdditionalParams: ''
- ]
+def runWithErrorHandling(def jobConfig, Closure build) {
try {
- timeout(time:30, unit: 'MINUTES', activity: true) {
-
- stage('Init') {
- checkout scm
- sh "git clean -fdx"
- def url = sh(returnStdout: true, script: 'git config remote.origin.url').trim()
- jobConfig.repoName = url.substring(url.lastIndexOf('/') + 1).replace('.git', '');
- if ( fileExists('.sling-module.json') ) {
- overrides = readJSON file: '.sling-module.json'
- echo "Jenkins overrides: ${overrides.jenkins}"
- overrides.jenkins.each { key,value ->
- jobConfig[key] = value;
- }
- }
- echo "Final job config: ${jobConfig}"
- }
-
- stage('Configure Job') {
- def upstreamProjectsCsv = jobConfig.upstreamProjects ?
- jsonArrayToCsv(jobConfig.upstreamProjects) : ''
- def jobTriggers = []
- if ( env.BRANCH_NAME == 'master' )
- jobTriggers.add(cron(jobConfig.rebuildFrequency))
- if ( upstreamProjectsCsv )
- jobTriggers.add(upstream(upstreamProjects: upstreamProjectsCsv, threshold: hudson.model.Result.SUCCESS))
-
- properties([
- pipelineTriggers(jobTriggers),
- buildDiscarder(logRotator(numToKeepStr: '10'))
- ])
- }
-
- build.call(jobConfig)
- }
+ build.call()
// exception handling copied from https://github.com/apache/maven-jenkins-lib/blob/d6c76aaea9df19ad88439eba4f9d1ad6c9e272bd/vars/asfMavenTlpPlgnBuild.groovy
} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
// this ambiguous condition means a user probably aborted
diff --git a/vars/slingOsgiBundleBuild.groovy b/vars/slingOsgiBundleBuild.groovy
index 45ede41..3ee129b 100644
--- a/vars/slingOsgiBundleBuild.groovy
+++ b/vars/slingOsgiBundleBuild.groovy
@@ -9,70 +9,103 @@ def call(Map params = [:]) {
githubCredentialsId: 'sling-github-token'
]
+ def helper = new SlingJenkinsHelper()
+ def jobConfig = [
+ jdks: [8],
+ upstreamProjects: [],
+ archivePatterns: [],
+ mavenGoal: '',
+ additionalMavenParams: '',
+ rebuildFrequency: '@weekly',
+ enabled: true,
+ emailRecipients: [],
+ sonarQubeEnabled: true,
+ sonarQubeUseAdditionalMavenParams: true,
+ sonarQubeAdditionalParams: ''
+ ]
+
node(globalConfig.mainNodeLabel) {
+ timeout(time:30, unit: 'MINUTES', activity: true) {
+ stage('Init') {
+ checkout scm
+ sh "git clean -fdx"
+ def url = sh(returnStdout: true, script: 'git config remote.origin.url').trim()
+ jobConfig.repoName = url.substring(url.lastIndexOf('/') + 1).replace('.git', '');
+ if ( fileExists('.sling-module.json') ) {
+ overrides = readJSON file: '.sling-module.json'
+ echo "Jenkins overrides: ${overrides.jenkins}"
+ overrides.jenkins.each { key,value ->
+ jobConfig[key] = value;
+ }
+ }
+ echo "Final job config: ${jobConfig}"
+ }
+ }
+ }
+
+ if ( isOnMainBranch() ) {
+ node(globalConfig.mainNodeLabel) {
+ timeout(time:30, unit: 'MINUTES', activity: true) {
+ stage('Configure Job') {
+ def upstreamProjectsCsv = jobConfig.upstreamProjects ?
+ jsonArrayToCsv(jobConfig.upstreamProjects) : ''
+ def jobTriggers = []
+ if ( env.BRANCH_NAME == 'master' )
+ jobTriggers.add(cron(jobConfig.rebuildFrequency))
+ if ( upstreamProjectsCsv )
+ jobTriggers.add(upstream(upstreamProjects: upstreamProjectsCsv, threshold: hudson.model.Result.SUCCESS))
+
+ properties([
+ pipelineTriggers(jobTriggers),
+ buildDiscarder(logRotator(numToKeepStr: '10'))
+ ])
+ }
+ }
+ }
+ }
- def helper = new SlingJenkinsHelper()
-
- helper.runWithErrorHandling({ jobConfig ->
- if ( jobConfig.enabled ) {
-
- // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
- // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
- def isReferenceStage = true
+ if ( jobConfig.enabled ) {
+ helper.runWithErrorHandling(jobConfig, {
- jobConfig.jdks.each { jdkVersion ->
- stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
- stageDefinition.call()
- isReferenceStage = false
- currentBuild.result = "SUCCESS"
+ // do a quick sanity check first without tests
+ // mvn clean compile
+ node(globalConfig.mainNodeLabel) {
+ stage("Sanity Check") {
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(8, globalConfig), // TODO: use Java version from first Job Config
+ publisherStrategy: 'EXPLICIT') {
+ sh "mvn clean compile ${additionalMavenParams(jobConfig)}"
+ }
}
+ }
- // this might fail if there are no jdks defined, but that's always an error
- // also, we don't activate any Maven publisher since we don't want this part of the
- // build tracked, but using withMaven(...) allows us to easily reuse the same
- // Maven and JDK versions
- def additionalMavenParams = additionalMavenParams(jobConfig)
- def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
+ // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
+ // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
+ def isReferenceStage = true
- if ( jobConfig.sonarQubeEnabled ) {
- stage('SonarCloud') {
- // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
- def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
- if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
- sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
- }
- // Params are different if it's a PR or if it's not
- // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
- if ( isPrBuild ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
- } else if ( env.BRANCH_NAME != "master" ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
- }
- static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
- // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
- withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
- // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
- withMaven(maven: globalConfig.mvnVersion,
- jdk: jenkinsJdkLabel(11, globalConfig),
- publisherStrategy: 'EXPLICIT') {
- try {
- sh "mvn -U clean verify ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams} -Pci"
- } catch ( Exception e ) {
- // TODO - we should check the actual failure cause here, but see
- // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
- echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
- currentBuild.result = 'UNSTABLE'
- }
- }
- }
+ // contains the label as key and a closure to execute as value
+ def stepsMap = [:]
+ // parallel execution of all build jobs (reference potentially including SonarQube)
+ jobConfig.jdks.each { jdkVersion ->
+ stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
+ stepsMap["$jdkVersion"] = stageDefinition
+ isReferenceStage = false
+ currentBuild.result = "SUCCESS"
+ }
+
+ parallel stepsMap
+
+ // last stage is deploy
+ if ( shouldDeploy() ) {
+ node(globalConfig.mainNodeLabel) {
+ stage("Deploy to Nexus") {
+ deployToNexus()
}
- } else {
- echo "SonarQube execution is disabled"
}
- } else {
- echo "Job is disabled, not building"
}
})
+ } else {
+ echo "Job is disabled, not building"
}
}
@@ -105,7 +138,15 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
if ( notMaster || !isSnapshot ) {
goal = "verify"
echo "Maven goal set to ${goal} since branch is not master ( ${env.BRANCH_NAME} ) or version is not snapshot ( ${mavenVersion} )"
- }
+ } else {
+ String localRepoPath = "${env.WORKSPACE}/local-snapshots-dir"
+ // Make sure the directory is wiped.
+ dir(localRepoPath) {
+ deleteDir()
+ }
+ // main build with IT for properly calculating coverage
+ additionalMavenParams = "${additionalMavenParams} -DaltDeploymentRepository=snapshot-repo::default::file:${localRepoPath}"
+ }
}
def invocation = {
@@ -122,21 +163,105 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
if ( isReferenceStage && jobConfig.archivePatterns ) {
archiveArtifacts(artifacts: SlingJenkinsHelper.jsonArrayToCsv(jobConfig.archivePatterns), allowEmptyArchive: true)
}
+ if ( isReferenceStage && shouldDeploy ) {
+ // Stash the build results so we can deploy them on another node
+ stash name: 'local-snapshots-dir', includes: 'local-snapshots-dir/**'
+ }
}
def branchConfig = jobConfig?.branches?."$env.BRANCH_NAME" ?: [:]
- if ( branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel )
- invocation = wrapInNode(invocation,branchConfig.nodeLabel)
-
return {
- stage("Build (Java ${jdkVersion}, ${goal})") {
- invocation.call()
+ wrapInNode(branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel, {
+ timeout(time: 30, unit: 'MINUTES') {
+ stage("Build (Java ${jdkVersion}, ${goal})") {
+ invocation.call()
+ }
+ }
+ if ( isReferenceStage ) {
+ // SonarQube (must be wrapped in same node)
+ if ( jobConfig.sonarQubeEnabled ) {
+ stage('Analyse with SonarCloud') {
+ timeout(time: 30, unit: 'MINUTES') {
+ analyseWithSonarCloud(globalConfig)
+ }
+ }
+ }
+ }
+ })
+ }
+}
+
+def analyseWithSonarCloud(def globalConfig) {
+ // this might fail if there are no jdks defined, but that's always an error
+ // also, we don't activate any Maven publisher since we don't want this part of the
+ // build tracked, but using withMaven(...) allows us to easily reuse the same
+ // Maven and JDK versions
+ def additionalMavenParams = additionalMavenParams(jobConfig)
+ def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
+
+ // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
+ def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
+ if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
+ sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
+ }
+ // Params are different if it's a PR or if it's not
+ // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
+ if ( isPrBuild ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
+ } else if ( env.BRANCH_NAME != "master" ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
+ }
+ static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
+ // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
+ withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
+ // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(11, globalConfig),
+ publisherStrategy: 'EXPLICIT') {
+ try {
+ sh "mvn ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams}"
+ } catch ( Exception e ) {
+ // TODO - we should check the actual failure cause here, but see
+ // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
+ echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
+ currentBuild.result = 'UNSTABLE'
+ }
}
}
}
-def wrapInNode(Closure invocation, def nodeLabel) {
+def deployToNexus() {
+ node('nexus-deploy') {
+ timeout(60) {
+ echo "Running on node ${env.NODE_NAME}"
+ // first clear workspace
+ deleteDir()
+ // Nexus deployment needs pom.xml
+ checkout scm
+ // Unstash the previously stashed build results.
+ unstash name: 'local-snapshots-dir'
+ // https://www.mojohaus.org/wagon-maven-plugin/merge-maven-repos-mojo.html
+ String mavenArguments = "${wagonPluginGav}:merge-maven-repos -Dwagon.target=https://repository.apache.org/content/repositories/snapshots -Dwagon.targetId=apache.snapshots.https -Dwagon.source=file:${env.WORKSPACE}/local-snapshots-dir"
+ pipelineSupport.executeMaven(this, mavenArguments, false)
+ }
+ }
+}
+
+boolean shouldDeploy() {
+ // check branch name
+ if ( !isOnMainBranch() ) {
+ return false;
+ }
+ // check maven goals
+ // check version
+}
+
+boolean isOnMainBranch() {
+ return env.BRANCH_NAME == "master"
+}
+
+def wrapInNode(def nodeLabel, Closure invocation) {
return {
node(nodeLabel) {
checkout scm