You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2021/03/26 22:26:34 UTC
[cassandra-builds] branch trunk updated: Split unit and jvm dtests.
Move jvm dtests to DTest stage in the pipeline
This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7ed9679 Split unit and jvm dtests. Move jvm dtests to DTest stage in the pipeline
7ed9679 is described below
commit 7ed96792878ed87340bde0224db0923777490f32
Author: Mick Semb Wever <mc...@apache.org>
AuthorDate: Sat Mar 13 16:43:23 2021 +0100
Split unit and jvm dtests. Move jvm dtests to DTest stage in the pipeline
patch by Mick Semb Wever; reviewed by Tomek Ĺasica for CASSANDRA-16402
---
build-scripts/cassandra-test.sh | 49 ++++++++++++++++-----
jenkins-dsl/cassandra_job_dsl_seed.groovy | 27 ++++++++----
jenkins-dsl/cassandra_pipeline.groovy | 72 +++++++++++++++----------------
3 files changed, 92 insertions(+), 56 deletions(-)
diff --git a/build-scripts/cassandra-test.sh b/build-scripts/cassandra-test.sh
index b25b8bc..bfb3ea9 100755
--- a/build-scripts/cassandra-test.sh
+++ b/build-scripts/cassandra-test.sh
@@ -7,10 +7,19 @@ set -o nounset
# lists all tests for the specific test type
_list_tests() {
- local readonly classlistprefix="$1"
+ local -r classlistprefix="$1"
find "test/$classlistprefix" -name '*Test.java' | sed "s;^test/$classlistprefix/;;g"
}
+_split_tests() {
+ local -r split_chunk="$1"
+ if [[ "x" == "x${split_chunk}" ]] ; then
+ split -n r/1/1
+ else
+ split -n r/${split_chunk}
+ fi
+}
+
_timeout_for() {
grep "name=\"$1\"" build.xml | awk -F'"' '{print $4}'
}
@@ -32,9 +41,13 @@ _build_all_dtest_jars() {
}
_main() {
- local target="${1:-}"
- local java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
- local version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
+ # parameters
+ local -r target="${1:-}"
+ local -r split_chunk="${2:-}" # Optional: pass in chunk to test, formatted as "K/N" for the Kth chunk of N chunks
+
+ local -r java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+ local -r version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
+
if [ "$java_version" -ge 11 ]; then
export CASSANDRA_USE_JDK11=true
if ! grep -q CASSANDRA_USE_JDK11 build.xml ; then
@@ -78,26 +91,40 @@ _main() {
ant $target -Dtmp.dir="$(pwd)/tmp" -Dmaven.test.failure.ignore=true
;;
"test")
- ant testclasslist -Dtest.classlistfile=<( _list_tests "unit" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist="$( _list_tests "unit" | _split_tests "${split_chunk}")"
+ echo ${testlist}
+ ant testclasslist -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
;;
"test-cdc")
- ant testclasslist-cdc -Dtest.classlistfile=<( _list_tests "unit" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "unit" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist-cdc -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
;;
"test-compression")
- ant testclasslist-compression -Dtest.classlistfile=<( _list_tests "unit" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "unit" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist-compression -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
;;
"test-burn")
- ant testclasslist -Dtest.classlistprefix=burn -Dtest.timeout=$(_timeout_for "test.burn.timeout") -Dtest.classlistfile=<( _list_tests "burn" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "burn" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist -Dtest.classlistprefix=burn -Dtest.timeout=$(_timeout_for "test.burn.timeout") -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
;;
"long-test")
- ant testclasslist -Dtest.classlistprefix=long -Dtest.timeout=$(_timeout_for "test.long.timeout") -Dtest.classlistfile=<( _list_tests "long" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "long" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist -Dtest.classlistprefix=long -Dtest.timeout=$(_timeout_for "test.long.timeout") -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
;;
"jvm-dtest")
- ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<( _list_tests "distributed" | grep -v "upgrade" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "distributed" | grep -v "upgrade" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
;;
"jvm-dtest-upgrade")
_build_all_dtest_jars
- ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<( _list_tests "distributed" | grep "upgrade" ) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ testlist=$( _list_tests "distributed" | grep "upgrade" | _split_tests "${split_chunk}")
+ echo ${testlist}
+ ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(${testlist}) -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
;;
*)
echo "unregconised \"$target\""
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 81f3d5d..24bf758 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -75,13 +75,12 @@ if(binding.hasVariable("MAX_JOB_HOURS")) {
}
// how many splits are dtest jobs matrixed into
+def testSplits = 8
def dtestSplits = 64
def dtestLargeSplits = 8
-if(binding.hasVariable("DTEST_SPLITS")) {
- dtestSplits = "${DTEST_SPLITS}"
-}
-if(binding.hasVariable("DTEST_LARGE_SPLITS")) {
- dtestLargeSplits = "${DTEST_LARGE_SPLITS}"
+
+def isSplittableTest(targetName) {
+ return targetName == 'test' || targetName == 'test-cdc' || targetName == 'test-compression' || targetName == 'test-burn' || targetName == 'long-test' || targetName == 'jvm-dtest' || targetName == 'jvm-dtest-upgrade';
}
////////////////////////////////////////////////////////////
@@ -523,6 +522,11 @@ cassandraBranches.each {
disabled(false)
using('Cassandra-template-test')
axes {
+ if (isSplittableTest(targetName)) {
+ List<String> values = new ArrayList<String>()
+ (1..testSplits).each { values << it.toString() }
+ text('split', values)
+ }
// jvm-dtest-upgrade would require mixed JDK compilations to support JDK11+
if (branchName == 'trunk' && targetName != 'jvm-dtest-upgrade') {
jdk(jdkLabel,'jdk_11_latest')
@@ -540,7 +544,7 @@ cassandraBranches.each {
}
steps {
shell("""
- ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} ;
+ ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} \${split}/${testSplits} ;
find build/test/logs -type f -name "*.log" | xargs xz -qq ;
./cassandra-builds/build-scripts/cassandra-test-report.sh ;
xz TESTS-TestSuites.xml
@@ -650,7 +654,7 @@ cassandraBranches.each {
}
properties {
githubProjectUrl(githubRepo)
- priorityJobProperty {
+ priorityJobProperty {
useJobPriority(true)
priority(1)
}
@@ -787,6 +791,11 @@ testTargets.each {
description(jobDescription)
concurrentBuild()
axes {
+ if (isSplittableTest(targetName)) {
+ List<String> values = new ArrayList<String>()
+ (1..testSplits).each { values << it.toString() }
+ text('split', values)
+ }
jdk(jdkLabel,'jdk_11_latest')
if (use_arm64_test_label()) {
label('label', slaveLabel, slaveArm64Label)
@@ -847,7 +856,7 @@ testTargets.each {
echo "Cassandra-devbranch-${targetName} cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head
""")
shell("""
- ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} ;
+ ./cassandra-builds/build-scripts/cassandra-test.sh ${targetName} \${split}/${testSplits} ;
find build/test/logs -type f -name "*.log" | xargs xz -qq ;
./cassandra-builds/build-scripts/cassandra-test-report.sh ;
xz TESTS-TestSuites.xml
@@ -979,7 +988,7 @@ archs.each {
""")
if (arch == "-arm64") {
shell("""
- # docker image has to be built on arm64 (as they are not published to dockerhub)
+ # docker image has to be built on arm64 (they are not currently published to dockerhub)
cd cassandra-builds/docker/testing ;
docker build -t \$DOCKER_IMAGE:latest -f ubuntu2004_j11.docker .
""")
diff --git a/jenkins-dsl/cassandra_pipeline.groovy b/jenkins-dsl/cassandra_pipeline.groovy
index da306ff..d852655 100644
--- a/jenkins-dsl/cassandra_pipeline.groovy
+++ b/jenkins-dsl/cassandra_pipeline.groovy
@@ -62,42 +62,6 @@ pipeline {
}
}
}
- stage('jvm-dtest') {
- steps {
- script {
- jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", parameters: [string(name: 'REPO', value: params.REPO), string(name: 'BRANCH', value: params.BRANCH)], propagate: false
- if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures')
- if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE'
- }
- }
- post {
- always {
- warnError('missing test xml files') {
- script {
- copyTestResults('jvm-dtest', jvm_dtest.getNumber())
- }
- }
- }
- }
- }
- stage('jvm-dtest-upgrade') {
- steps {
- script {
- jvm_dtest_upgrade = build job: "${env.JOB_NAME}-jvm-dtest-upgrade", parameters: [string(name: 'REPO', value: params.REPO), string(name: 'BRANCH', value: params.BRANCH)], propagate: false
- if (jvm_dtest_upgrade.result != 'SUCCESS') unstable('jvm-dtest-upgrade failures')
- if (jvm_dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE'
- }
- }
- post {
- always {
- warnError('missing test xml files') {
- script {
- copyTestResults('jvm-dtest-upgrade', jvm_dtest_upgrade.getNumber())
- }
- }
- }
- }
- }
stage('units') {
steps {
script {
@@ -210,6 +174,42 @@ pipeline {
}
stage('Distributed Test') {
parallel {
+ stage('jvm-dtest') {
+ steps {
+ script {
+ jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", parameters: [string(name: 'REPO', value: params.REPO), string(name: 'BRANCH', value: params.BRANCH)], propagate: false
+ if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures')
+ if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE'
+ }
+ }
+ post {
+ always {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('jvm-dtest', jvm_dtest.getNumber())
+ }
+ }
+ }
+ }
+ }
+ stage('jvm-dtest-upgrade') {
+ steps {
+ script {
+ jvm_dtest_upgrade = build job: "${env.JOB_NAME}-jvm-dtest-upgrade", parameters: [string(name: 'REPO', value: params.REPO), string(name: 'BRANCH', value: params.BRANCH)], propagate: false
+ if (jvm_dtest_upgrade.result != 'SUCCESS') unstable('jvm-dtest-upgrade failures')
+ if (jvm_dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE'
+ }
+ }
+ post {
+ always {
+ warnError('missing test xml files') {
+ script {
+ copyTestResults('jvm-dtest-upgrade', jvm_dtest_upgrade.getNumber())
+ }
+ }
+ }
+ }
+ }
stage('dtest') {
steps {
script {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org