You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2020/08/05 14:14:10 UTC
[cassandra-builds] branch master updated: Apply matrix splits to
all dtest jenkins jobs
This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/master by this push:
new 6fc260a Apply matrix splits to all dtest jenkins jobs
6fc260a is described below
commit 6fc260ae265e7c8f8c29445264ae143e1693cf14
Author: mck <mc...@apache.org>
AuthorDate: Fri Jul 31 16:25:00 2020 +0200
Apply matrix splits to all dtest jenkins jobs
patch by Mick Semb Wever; reviewed by Eduard Tudenhöfner for CASSANDRA-16006
---
jenkins-dsl/cassandra_job_dsl_seed.groovy | 102 ++++++------------------------
1 file changed, 19 insertions(+), 83 deletions(-)
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 619ddd5..4047963 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -68,6 +68,12 @@ if(binding.hasVariable("MAX_JOB_HOURS")) {
maxJobHours = "${MAX_JOB_HOURS}"
}
+// how many splits are dtest jobs matrixed into
+def dtestSplits = 64
+if(binding.hasVariable("DTEST_SPLITS")) {
+ dtestSplits = "${DTEST_SPLITS}"
+}
+
////////////////////////////////////////////////////////////
//
// Job Templates
@@ -224,67 +230,6 @@ job('Cassandra-template-test') {
/**
* Dtest template
*/
-job('Cassandra-template-dtest') {
- disabled(true)
- description(jobDescription)
- concurrentBuild()
- jdk(jdkLabel)
- label(slaveLabel)
- compressBuildLog()
- logRotator {
- numToKeep(10)
- artifactNumToKeep(10)
- }
- wrappers {
- timeout {
- noActivity(1200)
- }
- timestamps()
- }
- properties {
- githubProjectUrl(githubRepo)
- priority(5)
- }
- scm {
- git {
- remote {
- url(mainRepo)
- }
- branch('*/null')
- extensions {
- cleanAfterCheckout()
- }
- }
- }
- steps {
- buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; git clone --depth 1 --single-branch ${dtestRepo}")
- }
- publishers {
- archiveArtifacts {
- pattern('**/test_stdout.txt,**/nosetests.xml,**/ccm_logs.tar.xz')
- allowEmpty()
- fingerprint()
- }
- archiveJunit('nosetests.xml') {
- testDataPublishers {
- publishTestStabilityData()
- }
- }
- postBuildTask {
- task('.', """
- echo "Cleaning project…"; git clean -xdff ;
- echo "Pruning docker…" ; if pgrep -af jenkinscommand.sh; then docker system prune -f --filter 'until=${maxJobHours}h'; else docker system prune -f --volumes ; fi;
- echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs ../../* ;
- echo "Cleaning tmp…";
- find . -type d -name tmp -delete 2>/dev/null ;
- find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null
- """)
- }
- }
-}
-
-// wip – parallelise dtests, starting with just upgrade dtests
matrixJob('Cassandra-template-dtest-matrix') {
disabled(true)
description(jobDescription)
@@ -487,15 +432,13 @@ cassandraBranches.each {
// Skip dtest-offheap on cassandra-3.0 branch
if ((targetName == 'dtest-offheap') && (branchName == 'cassandra-3.0')) {
println("Skipping ${targetName} on branch ${branchName}")
- } else if (targetName == 'dtest-upgrade') {
- // wip – parallelise dtests, starting with just upgrade dtests
+ } else {
matrixJob("${jobNamePrefix}-${targetName}") {
disabled(false)
using('Cassandra-template-dtest-matrix')
axes {
- splits = 64
List<String> values = new ArrayList<String>()
- (1..splits).each { values << it.toString() }
+ (1..dtestSplits).each { values << it.toString() }
text('split', values)
label('label', slaveLabel)
}
@@ -503,21 +446,7 @@ cassandraBranches.each {
node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
}
steps {
- shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits}")
- }
- }
- } else {
- job("${jobNamePrefix}-${targetName}") {
- disabled(false)
- using('Cassandra-template-dtest')
- if (targetName == 'dtest-large') {
- label(largeSlaveLabel)
- }
- configure { node ->
- node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
- }
- steps {
- shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName}")
+ shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${dtestSplits}")
}
}
}
@@ -725,9 +654,10 @@ testTargets.each {
dtestTargets.each {
def targetName = it
- job("Cassandra-devbranch-${targetName}") {
+ matrixJob("Cassandra-devbranch-${targetName}") {
description(jobDescription)
concurrentBuild()
+ compressBuildLog()
jdk(jdkLabel)
if (targetName == 'dtest-large') {
label(largeSlaveLabel)
@@ -752,9 +682,15 @@ dtestTargets.each {
stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout')
stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests')
}
+ axes {
+ List<String> values = new ArrayList<String>()
+ (1..dtestSplits).each { values << it.toString() }
+ text('split', values)
+ label('label', slaveLabel)
+ }
properties {
githubProjectUrl(githubRepo)
- priority(5)
+ priority(6)
}
scm {
git {
@@ -770,7 +706,7 @@ dtestTargets.each {
steps {
buildDescription('', buildDescStr)
shell("git clean -xdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo}")
- shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName}")
+ shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${dtestSplits}")
}
publishers {
archiveArtifacts {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org