You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2021/04/05 15:47:04 UTC
[kafka] 02/02: KAFKA-12614: Use Jenkinsfile for trunk and release
branch builds (#10473)
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch 2.8
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 0ba00600c674a2079ff29f9c7c71c76bc40a2b63
Author: Ismael Juma <is...@juma.me.uk>
AuthorDate: Mon Apr 5 08:42:04 2021 -0700
KAFKA-12614: Use Jenkinsfile for trunk and release branch builds (#10473)
* Run all JDK/Scala version combinations for trunk/release branch builds.
* Only retry failures in PR builds for now (we can remove this distinction if/when
we report flaky failures as described in KAFKA-12216).
* Disable concurrent builds
* Send email to dev list on build failure
* Use triple double quotes in `doValidation` since we use string interpolation
for `SCALA_VERSION`.
* Update release.py to output new `Unit/integration tests` Jenkins link
Reviewers: Gwen Shapira <cs...@gmail.com>, David Arthur <mu...@gmail.com>
---
Jenkinsfile | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++++-------
release.py | 2 +-
2 files changed, 121 insertions(+), 17 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 5579887..f9110c6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,19 +26,26 @@ def setupGradle() {
}
def doValidation() {
- sh '''
+ sh """
./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \
--profile --no-daemon --continue -PxmlSpotBugsReport=true
- '''
+ """
}
-def doTest(target = "unitTest integrationTest") {
- sh """
- ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
- --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
- -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5
- """
+def isChangeRequest(env) {
+ env.CHANGE_ID != null && !env.CHANGE_ID.isEmpty()
+}
+
+def retryFlagsString(env) {
+ if (isChangeRequest(env)) " -PmaxTestRetries=1 -PmaxTestRetryFailures=5"
+ else ""
+}
+
+def doTest(env, target = "unitTest integrationTest") {
+ sh """./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
+ --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
+ -PignoreFailures=true -PmaxParallelForks=2""" + retryFlagsString(env)
junit '**/build/test-results/**/TEST-*.xml'
}
@@ -95,10 +102,16 @@ def tryStreamsArchetype() {
pipeline {
agent none
+
+ options {
+ disableConcurrentBuilds()
+ }
+
stages {
stage('Build') {
parallel {
- stage('JDK 8') {
+
+ stage('JDK 8 and Scala 2.12') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_1.8_latest'
@@ -114,12 +127,12 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
tryStreamsArchetype()
}
}
- stage('JDK 11') {
+ stage('JDK 11 and Scala 2.13') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_11_latest'
@@ -134,12 +147,12 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
echo 'Skipping Kafka Streams archetype test for Java 11'
}
}
-
- stage('JDK 15') {
+
+ stage('JDK 15 and Scala 2.13') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_15_latest'
@@ -154,7 +167,7 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
echo 'Skipping Kafka Streams archetype test for Java 15'
}
}
@@ -172,11 +185,102 @@ pipeline {
setupGradle()
doValidation()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
- doTest('unitTest')
+ doTest(env, 'unitTest')
}
echo 'Skipping Kafka Streams archetype test for ARM build'
}
}
+
+ // To avoid excessive Jenkins resource usage, we only run the stages
+ // above at the PR stage. The ones below are executed after changes
+ // are pushed to trunk and/or release branches. We achieve this via
+ // the `when` clause.
+
+ stage('JDK 8 and Scala 2.13') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_1.8_latest'
+ maven 'maven_3_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.13
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ tryStreamsArchetype()
+ }
+ }
+
+ stage('JDK 11 and Scala 2.12') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_11_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.12
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ echo 'Skipping Kafka Streams archetype test for Java 11'
+ }
+ }
+
+ stage('JDK 15 and Scala 2.12') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_15_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.12
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ echo 'Skipping Kafka Streams archetype test for Java 15'
+ }
+ }
+ }
+ }
+ }
+
+ post {
+ always {
+ script {
+ if (!isChangeRequest(env)) {
+ step([$class: 'Mailer',
+ notifyEveryUnstableBuild: true,
+ recipients: "dev@kafka.apache.org",
+ sendToIndividuals: false])
+ }
}
}
}
diff --git a/release.py b/release.py
index 94bb173..ef00ec0 100755
--- a/release.py
+++ b/release.py
@@ -731,7 +731,7 @@ https://kafka.apache.org/%(docs_version)s/documentation.html
https://kafka.apache.org/%(docs_version)s/protocol.html
* Successful Jenkins builds for the %(dev_branch)s branch:
-Unit/integration tests: https://builds.apache.org/job/kafka-%(dev_branch)s-jdk8/<BUILD NUMBER>/
+Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s/<BUILD NUMBER>/
System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s/<BUILD_NUMBER>/
/**************************************