You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2021/04/05 15:47:02 UTC
[kafka] branch 2.8 updated (892ac57 -> 0ba0060)
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a change to branch 2.8
in repository https://gitbox.apache.org/repos/asf/kafka.git.
from 892ac57 MINOR: remove KTable.to from the docs (#10464)
new 5d8f461 KAFKA-10759 Add ARM build stage (#9992)
new 0ba0060 KAFKA-12614: Use Jenkinsfile for trunk and release branch builds (#10473)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Jenkinsfile | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------
release.py | 2 +-
2 files changed, 139 insertions(+), 16 deletions(-)
[kafka] 02/02: KAFKA-12614: Use Jenkinsfile for trunk and release
branch builds (#10473)
Posted by ij...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch 2.8
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 0ba00600c674a2079ff29f9c7c71c76bc40a2b63
Author: Ismael Juma <is...@juma.me.uk>
AuthorDate: Mon Apr 5 08:42:04 2021 -0700
KAFKA-12614: Use Jenkinsfile for trunk and release branch builds (#10473)
* Run all JDK/Scala version combinations for trunk/release branch builds.
* Only retry failures in PR builds for now (we can remove this distinction if/when
we report flaky failures as described in KAFKA-12216).
* Disable concurrent builds
* Send email to dev list on build failure
* Use triple double quotes in `doValidation` since we use string interpolation
for `SCALA_VERSION`.
* Update release.py to output new `Unit/integration tests` Jenkins link
Reviewers: Gwen Shapira <cs...@gmail.com>, David Arthur <mu...@gmail.com>
---
Jenkinsfile | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++++-------
release.py | 2 +-
2 files changed, 121 insertions(+), 17 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 5579887..f9110c6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,19 +26,26 @@ def setupGradle() {
}
def doValidation() {
- sh '''
+ sh """
./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \
spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \
--profile --no-daemon --continue -PxmlSpotBugsReport=true
- '''
+ """
}
-def doTest(target = "unitTest integrationTest") {
- sh """
- ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
- --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
- -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5
- """
+def isChangeRequest(env) {
+ env.CHANGE_ID != null && !env.CHANGE_ID.isEmpty()
+}
+
+def retryFlagsString(env) {
+ if (isChangeRequest(env)) " -PmaxTestRetries=1 -PmaxTestRetryFailures=5"
+ else ""
+}
+
+def doTest(env, target = "unitTest integrationTest") {
+ sh """./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
+ --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
+ -PignoreFailures=true -PmaxParallelForks=2""" + retryFlagsString(env)
junit '**/build/test-results/**/TEST-*.xml'
}
@@ -95,10 +102,16 @@ def tryStreamsArchetype() {
pipeline {
agent none
+
+ options {
+ disableConcurrentBuilds()
+ }
+
stages {
stage('Build') {
parallel {
- stage('JDK 8') {
+
+ stage('JDK 8 and Scala 2.12') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_1.8_latest'
@@ -114,12 +127,12 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
tryStreamsArchetype()
}
}
- stage('JDK 11') {
+ stage('JDK 11 and Scala 2.13') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_11_latest'
@@ -134,12 +147,12 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
echo 'Skipping Kafka Streams archetype test for Java 11'
}
}
-
- stage('JDK 15') {
+
+ stage('JDK 15 and Scala 2.13') {
agent { label 'ubuntu' }
tools {
jdk 'jdk_15_latest'
@@ -154,7 +167,7 @@ pipeline {
steps {
setupGradle()
doValidation()
- doTest()
+ doTest(env)
echo 'Skipping Kafka Streams archetype test for Java 15'
}
}
@@ -172,11 +185,102 @@ pipeline {
setupGradle()
doValidation()
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
- doTest('unitTest')
+ doTest(env, 'unitTest')
}
echo 'Skipping Kafka Streams archetype test for ARM build'
}
}
+
+ // To avoid excessive Jenkins resource usage, we only run the stages
+ // above at the PR stage. The ones below are executed after changes
+ // are pushed to trunk and/or release branches. We achieve this via
+ // the `when` clause.
+
+ stage('JDK 8 and Scala 2.13') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_1.8_latest'
+ maven 'maven_3_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.13
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ tryStreamsArchetype()
+ }
+ }
+
+ stage('JDK 11 and Scala 2.12') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_11_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.12
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ echo 'Skipping Kafka Streams archetype test for Java 11'
+ }
+ }
+
+ stage('JDK 15 and Scala 2.12') {
+ when {
+ not { changeRequest() }
+ beforeAgent true
+ }
+ agent { label 'ubuntu' }
+ tools {
+ jdk 'jdk_15_latest'
+ }
+ options {
+ timeout(time: 8, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.12
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ doTest(env)
+ echo 'Skipping Kafka Streams archetype test for Java 15'
+ }
+ }
+ }
+ }
+ }
+
+ post {
+ always {
+ script {
+ if (!isChangeRequest(env)) {
+ step([$class: 'Mailer',
+ notifyEveryUnstableBuild: true,
+ recipients: "dev@kafka.apache.org",
+ sendToIndividuals: false])
+ }
}
}
}
diff --git a/release.py b/release.py
index 94bb173..ef00ec0 100755
--- a/release.py
+++ b/release.py
@@ -731,7 +731,7 @@ https://kafka.apache.org/%(docs_version)s/documentation.html
https://kafka.apache.org/%(docs_version)s/protocol.html
* Successful Jenkins builds for the %(dev_branch)s branch:
-Unit/integration tests: https://builds.apache.org/job/kafka-%(dev_branch)s-jdk8/<BUILD NUMBER>/
+Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s/<BUILD NUMBER>/
System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s/<BUILD_NUMBER>/
/**************************************
[kafka] 01/02: KAFKA-10759 Add ARM build stage (#9992)
Posted by ij...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ijuma pushed a commit to branch 2.8
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 5d8f4614d3732b401525f54113015faeea83e121
Author: David Arthur <mu...@gmail.com>
AuthorDate: Wed Mar 3 15:57:27 2021 -0500
KAFKA-10759 Add ARM build stage (#9992)
Only validation and unit test stages are enabled
Co-authored-by: Peng.Lei <73...@users.noreply.github.com>
---
Jenkinsfile | 27 +++++++++++++++++++++++----
1 file changed, 23 insertions(+), 4 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 87840d5..5579887 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -33,12 +33,12 @@ def doValidation() {
'''
}
-def doTest() {
- sh '''
- ./gradlew -PscalaVersion=$SCALA_VERSION unitTest integrationTest \
+def doTest(target = "unitTest integrationTest") {
+ sh """
+ ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \
--profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \
-PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5
- '''
+ """
junit '**/build/test-results/**/TEST-*.xml'
}
@@ -158,6 +158,25 @@ pipeline {
echo 'Skipping Kafka Streams archetype test for Java 15'
}
}
+
+ stage('ARM') {
+ agent { label 'arm4' }
+ options {
+ timeout(time: 2, unit: 'HOURS')
+ timestamps()
+ }
+ environment {
+ SCALA_VERSION=2.12
+ }
+ steps {
+ setupGradle()
+ doValidation()
+ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+ doTest('unitTest')
+ }
+ echo 'Skipping Kafka Streams archetype test for ARM build'
+ }
+ }
}
}
}