You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by kw...@apache.org on 2022/06/15 07:47:52 UTC
[sling-tooling-jenkins] 01/01: SLING-9948 parallelize steps
This is an automated email from the ASF dual-hosted git repository.
kwin pushed a commit to branch feature/parallel-builds
in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
commit 56281bc3be48f0a0f3134a8b7e5c874a47ebf2e3
Author: Konrad Windszus <kw...@apache.org>
AuthorDate: Wed Jun 15 09:47:47 2022 +0200
SLING-9948 parallelize steps
reuse build artifacts for SonarCloud analysis
deploy only if every other previous stage was successfully executed
---
vars/slingOsgiBundleBuild.groovy | 204 ++++++++++++++++++++++++++-------------
1 file changed, 137 insertions(+), 67 deletions(-)
diff --git a/vars/slingOsgiBundleBuild.groovy b/vars/slingOsgiBundleBuild.groovy
index 45ede41..875ef3d 100644
--- a/vars/slingOsgiBundleBuild.groovy
+++ b/vars/slingOsgiBundleBuild.groovy
@@ -9,71 +9,51 @@ def call(Map params = [:]) {
githubCredentialsId: 'sling-github-token'
]
- node(globalConfig.mainNodeLabel) {
-
- def helper = new SlingJenkinsHelper()
-
- helper.runWithErrorHandling({ jobConfig ->
- if ( jobConfig.enabled ) {
-
- // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
- // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
- def isReferenceStage = true
-
- jobConfig.jdks.each { jdkVersion ->
- stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
- stageDefinition.call()
- isReferenceStage = false
- currentBuild.result = "SUCCESS"
+ def helper = new SlingJenkinsHelper()
+
+ helper.runWithErrorHandling({ jobConfig ->
+ if ( jobConfig.enabled ) {
+
+ // do a quick sanity check first without tests
+ // mvn clean compile
+ node(globalConfig.mainNodeLabel) {
+ stage("Sanity Check") {
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(8, globalConfig), // TODO: use Java version from first Job Config
+ publisherStrategy: 'EXPLICIT') ) {
+ sh "mvn clean compile ${additionalMavenParams}"
+ }
}
+ }
- // this might fail if there are no jdks defined, but that's always an error
- // also, we don't activate any Maven publisher since we don't want this part of the
- // build tracked, but using withMaven(...) allows us to easily reuse the same
- // Maven and JDK versions
- def additionalMavenParams = additionalMavenParams(jobConfig)
- def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
-
- if ( jobConfig.sonarQubeEnabled ) {
- stage('SonarCloud') {
- // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
- def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
- if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
- sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
- }
- // Params are different if it's a PR or if it's not
- // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
- if ( isPrBuild ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
- } else if ( env.BRANCH_NAME != "master" ) {
- sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
- }
- static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
- // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
- withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
- // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
- withMaven(maven: globalConfig.mvnVersion,
- jdk: jenkinsJdkLabel(11, globalConfig),
- publisherStrategy: 'EXPLICIT') {
- try {
- sh "mvn -U clean verify ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams} -Pci"
- } catch ( Exception e ) {
- // TODO - we should check the actual failure cause here, but see
- // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
- echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
- currentBuild.result = 'UNSTABLE'
- }
- }
- }
+ // the reference build is always the first one, and the only one to deploy, archive artifacts, etc
+ // usually this is the build done with the oldest JDK version, to ensure maximum compatibility
+ def isReferenceStage = true
+
+ // contains the label as key and a closure to execute as value
+ def stepsMap = [:]
+ // parallel execution of all build jobs (reference potentially including SonarQube)
+ jobConfig.jdks.each { jdkVersion ->
+ stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage)
+ stepsMap["$jdkVersion"] = stageDefinition
+ isReferenceStage = false
+ currentBuild.result = "SUCCESS"
+ }
+
+ parallel stepsMap
+
+ // last stage is deploy
+ if ( shouldDeploy() ) {
+ node(globalConfig.mainNodeLabel) {
+ stage("Deploy to Nexus") {
+ deployToNexus()
}
- } else {
- echo "SonarQube execution is disabled"
}
- } else {
- echo "Job is disabled, not building"
}
- })
- }
+ } else {
+ echo "Job is disabled, not building"
+ }
+ })
}
def jenkinsJdkLabel(int jdkVersion, def globalConfig) {
@@ -105,7 +85,15 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
if ( notMaster || !isSnapshot ) {
goal = "verify"
echo "Maven goal set to ${goal} since branch is not master ( ${env.BRANCH_NAME} ) or version is not snapshot ( ${mavenVersion} )"
- }
+ } else {
+ String localRepoPath = "${env.WORKSPACE}/local-snapshots-dir"
+ // Make sure the directory is wiped.
+ dir(localRepoPath) {
+ deleteDir()
+ }
+ // main build with IT for properly calculating coverage
+ additionalMavenParams = "${additionalMavenParams} -DaltDeploymentRepository=snapshot-repo::default::file:${localRepoPath}"
+ }
}
def invocation = {
@@ -122,21 +110,103 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference
if ( isReferenceStage && jobConfig.archivePatterns ) {
archiveArtifacts(artifacts: SlingJenkinsHelper.jsonArrayToCsv(jobConfig.archivePatterns), allowEmptyArchive: true)
}
+ if ( isReferenceStage && shouldDeploy ) {
+ // Stash the build results so we can deploy them on another node
+ stash name: 'local-snapshots-dir', includes: 'local-snapshots-dir/**'
+ }
}
def branchConfig = jobConfig?.branches?."$env.BRANCH_NAME" ?: [:]
- if ( branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel )
- invocation = wrapInNode(invocation,branchConfig.nodeLabel)
-
return {
- stage("Build (Java ${jdkVersion}, ${goal})") {
- invocation.call()
+ wrapInNode(branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel), {
+ stage("Build (Java ${jdkVersion}, ${goal})") {
+ invocation.call()
+ }
+ if ( isReferenceStage ) {
+ // SonarQube (must be wrapped in same node)
+ if ( jobConfig.sonarQubeEnabled ) {
+ stage('Analyse with SonarCloud') {
+ timeout(60) {
+ analyseWithSonarCloud(globalConfig)
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+def analyseWithSonarCloud(def globalConfig) {
+ // this might fail if there are no jdks defined, but that's always an error
+ // also, we don't activate any Maven publisher since we don't want this part of the
+ // build tracked, but using withMaven(...) allows us to easily reuse the same
+ // Maven and JDK versions
+ def additionalMavenParams = additionalMavenParams(jobConfig)
+ def isPrBuild = env.BRANCH_NAME.startsWith("PR-")
+
+ // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually
+ def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}"
+ if ( jobConfig.sonarQubeUseAdditionalMavenParams ) {
+ sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}"
+ }
+ // Params are different if it's a PR or if it's not
+ // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853
+ if ( isPrBuild ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}"
+ } else if ( env.BRANCH_NAME != "master" ) {
+ sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}"
+ }
+ static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184'
+ // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token
+ withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) {
+ // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/)
+ withMaven(maven: globalConfig.mvnVersion,
+ jdk: jenkinsJdkLabel(11, globalConfig),
+ publisherStrategy: 'EXPLICIT') {
+ try {
+ sh "mvn ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams}"
+ } catch ( Exception e ) {
+ // TODO - we should check the actual failure cause here, but see
+ // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122
+ echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info."
+ currentBuild.result = 'UNSTABLE'
+ }
+ }
+ }
+}
+
+def deployToNexus() {
+ node('nexus-deploy') {
+ timeout(60) {
+ echo "Running on node ${env.NODE_NAME}"
+ // first clear workspace
+ deleteDir()
+ // Nexus deployment needs pom.xml
+ checkout scm
+ // Unstash the previously stashed build results.
+ unstash name: 'local-snapshots-dir'
+ // https://www.mojohaus.org/wagon-maven-plugin/merge-maven-repos-mojo.html
+ String mavenArguments = "${wagonPluginGav}:merge-maven-repos -Dwagon.target=https://repository.apache.org/content/repositories/snapshots -Dwagon.targetId=apache.snapshots.https -Dwagon.source=file:${env.WORKSPACE}/local-snapshots-dir"
+ pipelineSupport.executeMaven(this, mavenArguments, false)
}
}
}
-def wrapInNode(Closure invocation, def nodeLabel) {
+boolean shouldDeploy() {
+ // check branch name
+ if ( !isOnMainBranch() ) {
+ return false;
+ }
+ // check maven goals
+ // check version
+}
+
+boolean isOnMainBranch() {
+ return env.BRANCH_NAME == "master"
+}
+
+def wrapInNode(def nodeLabel, Closure invocation) {
return {
node(nodeLabel) {
checkout scm