You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/06/04 20:07:57 UTC
[bookkeeper] branch master updated: Improve precommit CI jobs on
handling PRs made to branches
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 9e937cf Improve precommit CI jobs on handling PRs made to branches
9e937cf is described below
commit 9e937cfc9da15521fff1946ad3470d1f35b3b016
Author: Sijie Guo <si...@apache.org>
AuthorDate: Mon Jun 4 13:07:42 2018 -0700
Improve precommit CI jobs on handling PRs made to branches
Descriptions of the changes in this PR:
*Motivation*
Current precommit CI jobs uses `master` as target branch for applying the pull requests.
However pull requests might be made against branches. e.g. #1469, precommit jobs can't run properly on those pull requests. This PR tends to address the problem.
*Solution*
Use `ghprbTargetBranch` as the build branch for precommit jobs.
Besides that, split the validation goals (e.g. checkstyle, apache-rat) from build/test goals (package, spotbugs), this reduces build time for different jdk versions and address problem when running checkstyle:check with other goals (e.g. deploy).
Author: Sijie Guo <si...@apache.org>
Reviewers: Enrico Olivelli <eo...@gmail.com>
This closes #1475 from sijie/run_precommit_jobs_over_target_branches
---
.test-infra/jenkins/common_job_properties.groovy | 40 ++++++++++++++--------
.../job_bookkeeper_postcommit_master_java8.groovy | 2 +-
.../job_bookkeeper_postcommit_master_java9.groovy | 2 +-
...bookkeeper_postcommit_validation_master.groovy} | 12 +++----
...ob_bookkeeper_precommit_integrationtests.groovy | 4 ++-
.../jenkins/job_bookkeeper_precommit_java8.groovy | 6 ++--
.../jenkins/job_bookkeeper_precommit_java9.groovy | 6 ++--
... => job_bookkeeper_precommit_validation.groovy} | 23 +++++--------
.../job_bookkeeper_release_nightly_snapshot.groovy | 2 +-
9 files changed, 54 insertions(+), 43 deletions(-)
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index 2b51f42..86162a8 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -43,7 +43,8 @@ class common_job_properties {
String branch = 'master',
String jdkVersion = 'JDK 1.8 (latest)',
int timeout = 200,
- String jenkinsExecutorLabel = 'ubuntu') {
+ String jenkinsExecutorLabel = 'ubuntu',
+ String branchVarName = '${sha1}') {
// GitHub project.
context.properties {
githubProjectUrl('https://github.com/apache/bookkeeper/')
@@ -55,7 +56,9 @@ class common_job_properties {
'https://github.com/apache/bookkeeper.git',
branch,
jenkinsExecutorLabel,
- timeout)
+ timeout,
+ jdkVersion,
+ branchVarName)
}
// Sets common top-level job properties. Accessed through one of the above
@@ -65,7 +68,8 @@ class common_job_properties {
String defaultBranch,
String jenkinsExecutorLabel,
int defaultTimeout,
- String jdkVersion = 'JDK 1.8 (latest)') {
+ String jdkVersion = 'JDK 1.8 (latest)',
+ String branchVarName = '${sha1}') {
// Set JDK version.
context.jdk(jdkVersion)
@@ -85,20 +89,23 @@ class common_job_properties {
refspec('+refs/heads/*:refs/remotes/origin/* ' +
'+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*')
}
- branch('${sha1}')
+ branch(branchVarName)
extensions {
cleanAfterCheckout()
}
}
}
- context.parameters {
- // This is a recommended setup if you want to run the job manually. The
- // ${sha1} parameter needs to be provided, and defaults to the main branch.
- stringParam(
- 'sha1',
- defaultBranch,
- 'Commit id or refname (eg: origin/pr/9/head) you want to build.')
+ // add the parameter when branch var name is `sha1`
+ if (branchVarName == '${sha1}') {
+ context.parameters {
+ // This is a recommended setup if you want to run the job manually. The
+ // ${sha1} parameter needs to be provided, and defaults to the main branch.
+ stringParam(
+ 'sha1',
+ defaultBranch,
+ 'Commit id or refname (eg: origin/pr/9/head) you want to build.')
+ }
}
context.wrappers {
@@ -120,7 +127,8 @@ class common_job_properties {
private static void setPullRequestBuildTrigger(context,
String commitStatusContext,
String successComment = '--none--',
- String prTriggerPhrase = '') {
+ String prTriggerPhrase = '',
+ boolean onlyMaster = false) {
context.triggers {
githubPullRequest {
admins(['asfbot'])
@@ -137,6 +145,9 @@ class common_job_properties {
triggerPhrase(prTriggerPhrase)
onlyTriggerPhrase()
}
+ if (onlyMaster) {
+ whiteListTargetBranches(['master'])
+ }
extensions {
commitStatus {
@@ -202,9 +213,10 @@ class common_job_properties {
// Sets common config for PreCommit jobs.
static void setPreCommit(context,
String commitStatusName,
- String successComment = '--none--') {
+ String successComment = '--none--',
+ boolean onlyMaster = false) {
// Set pull request build trigger.
- setPullRequestBuildTrigger(context, commitStatusName, successComment)
+ setPullRequestBuildTrigger(context, commitStatusName, successComment, '', onlyMaster)
}
// Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase
diff --git a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy b/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy
index ed13b8c..cb15c3f 100644
--- a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy
@@ -42,5 +42,5 @@ mavenJob('bookkeeper_postcommit_master_java8') {
common_job_properties.setMavenConfig(delegate)
// Maven build project.
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Ddistributedlog -Dstream -DstreamTests')
+ goals('clean package spotbugs:check -Ddistributedlog -Dstream -DstreamTests')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java9.groovy b/.test-infra/jenkins/job_bookkeeper_postcommit_master_java9.groovy
index 006255f..942fd09 100644
--- a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java9.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_postcommit_master_java9.groovy
@@ -42,5 +42,5 @@ mavenJob('bookkeeper_postcommit_master_java9') {
common_job_properties.setMavenConfig(delegate)
// Maven build project.
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Ddistributedlog -Dstream -DstreamTests')
+ goals('clean package spotbugs:check -Ddistributedlog -Dstream -DstreamTests')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy b/.test-infra/jenkins/job_bookkeeper_postcommit_validation_master.groovy
similarity index 79%
copy from .test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy
copy to .test-infra/jenkins/job_bookkeeper_postcommit_validation_master.groovy
index ed13b8c..f914f44 100644
--- a/.test-infra/jenkins/job_bookkeeper_postcommit_master_java8.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_postcommit_validation_master.groovy
@@ -18,9 +18,9 @@
import common_job_properties
-// This job runs the Java postcommit tests on Java 8
-mavenJob('bookkeeper_postcommit_master_java8') {
- description('Runs nightly build for bookkeeper in Java 8.')
+// This job runs the Java postcommit validation on master branch
+mavenJob('bookkeeper_postcommit_validation_master') {
+ description('Runs postcommit validation nightly for bookkeeper.')
// Set common parameters.
common_job_properties.setTopLevelMainJobProperties(
@@ -35,12 +35,12 @@ mavenJob('bookkeeper_postcommit_master_java8') {
// Allows triggering this build against pull requests.
common_job_properties.enablePhraseTriggeringFromPullRequest(
delegate,
- 'Java 8 Test',
- '/test-java8')
+ 'Postcommit Validation',
+ '/postcommit-validation')
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
// Maven build project.
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Ddistributedlog -Dstream -DstreamTests')
+ goals('clean apache-rat:check checkstyle:check package -Ddistributedlog -Dstream -DskipTests')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
index 8cd5c9a..9cd7635 100644
--- a/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_precommit_integrationtests.groovy
@@ -27,7 +27,9 @@ freeStyleJob('bookkeeper_precommit_integrationtests') {
delegate,
'master',
'JDK 1.8 (latest)',
- 120)
+ 120,
+ 'ubuntu',
+ '${ghprbTargetBranch}')
// Execute concurrent builds if necessary.
concurrentBuild()
diff --git a/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy b/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy
index 3586b35..28e7214 100644
--- a/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy
@@ -39,7 +39,9 @@ mavenJob('bookkeeper_precommit_pullrequest_java8') {
delegate,
'master',
'JDK 1.8 (latest)',
- 200)
+ 200,
+ 'ubuntu',
+ '${ghprbTargetBranch}')
// Sets that this is a PreCommit job.
common_job_properties.setPreCommit(delegate, 'Maven clean install (Java 8)')
@@ -48,5 +50,5 @@ mavenJob('bookkeeper_precommit_pullrequest_java8') {
common_job_properties.setMavenConfig(delegate)
// Maven build project
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Dstream')
+ goals('clean apache-rat:check package spotbugs:check -Dstream')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_precommit_java9.groovy b/.test-infra/jenkins/job_bookkeeper_precommit_java9.groovy
index 16a8449..120144e 100644
--- a/.test-infra/jenkins/job_bookkeeper_precommit_java9.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_precommit_java9.groovy
@@ -39,7 +39,9 @@ mavenJob('bookkeeper_precommit_pullrequest_java9') {
delegate,
'master',
'JDK 1.9 (latest)',
- 200)
+ 200,
+ 'ubuntu',
+ '${ghprbTargetBranch}')
// Sets that this is a PreCommit job.
common_job_properties.setPreCommit(delegate, 'Maven clean install (Java 9)')
@@ -48,5 +50,5 @@ mavenJob('bookkeeper_precommit_pullrequest_java9') {
common_job_properties.setMavenConfig(delegate)
// Maven build project
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Dstream')
+ goals('clean apache-rat:check package spotbugs:check -Dstream')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy b/.test-infra/jenkins/job_bookkeeper_precommit_validation.groovy
similarity index 64%
copy from .test-infra/jenkins/job_bookkeeper_precommit_java8.groovy
copy to .test-infra/jenkins/job_bookkeeper_precommit_validation.groovy
index 3586b35..48d6c3a 100644
--- a/.test-infra/jenkins/job_bookkeeper_precommit_java8.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_precommit_validation.groovy
@@ -18,18 +18,9 @@
import common_job_properties
-// This is the Java precommit which runs a maven install, and the current set of precommit tests.
-mavenJob('bookkeeper_precommit_pullrequest_java8') {
- description('precommit verification for pull requests of <a href="http://bookkeeper.apache.org">Apache BookKeeper</a> in Java 8.')
-
- // Temporary information gathering to see if full disks are causing the builds to flake
- preBuildSteps {
- shell("id")
- shell("ulimit -a")
- shell("pwd")
- shell("df -h")
- shell("ps aux")
- }
+// This is the Java precommit validation job that validates pull requests (e.g. checkstyle)
+mavenJob('bookkeeper_precommit_pullrequest_validation') {
+ description('precommit validation for pull requests of <a href="http://bookkeeper.apache.org">Apache BookKeeper</a>.')
// Execute concurrent builds if necessary.
concurrentBuild()
@@ -39,14 +30,16 @@ mavenJob('bookkeeper_precommit_pullrequest_java8') {
delegate,
'master',
'JDK 1.8 (latest)',
- 200)
+ 200,
+ 'ubuntu',
+ '${ghprbTargetBranch}')
// Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, 'Maven clean install (Java 8)')
+ common_job_properties.setPreCommit(delegate, 'Pull Request Validation', '--none--', true)
// Set Maven parameters.
common_job_properties.setMavenConfig(delegate)
// Maven build project
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Dstream')
+ goals('clean apache-rat:check checkstyle:check package -Ddistributedlog -Dstream -DskipTests')
}
diff --git a/.test-infra/jenkins/job_bookkeeper_release_nightly_snapshot.groovy b/.test-infra/jenkins/job_bookkeeper_release_nightly_snapshot.groovy
index 1a81444..fa2dd61 100644
--- a/.test-infra/jenkins/job_bookkeeper_release_nightly_snapshot.groovy
+++ b/.test-infra/jenkins/job_bookkeeper_release_nightly_snapshot.groovy
@@ -64,7 +64,7 @@ export MAVEN_OPTS=-Xmx2048m
common_job_properties.setMavenConfig(delegate)
// Maven build project.
- goals('clean apache-rat:check checkstyle:check package spotbugs:check -Dmaven.test.failure.ignore=true deploy -Ddistributedlog -Dstream -DstreamTests -Pdocker')
+ goals('clean package -Dmaven.test.failure.ignore=true deploy -Ddistributedlog -Dstream -DstreamTests -Pdocker')
}
// publish the docker images
--
To stop receiving notification emails like this one, please contact
sijie@apache.org.