You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2021/05/06 09:56:25 UTC
[cassandra-builds] 01/03: Split dockerised test runs into parallel
docker container runs
This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
commit 4ca24701e105226ca35da1adc7ff70b56a36fce2
Author: Mick Semb Wever <mc...@apache.org>
AuthorDate: Sun Apr 11 12:34:41 2021 +0200
Split dockerised test runs into parallel docker container runs
Improvements
- build arm64 docker images on-the-fly (they are not deployed anywhere yet)
- only pull the docker images if they are not found locally (saves time!)
- return internal exit codes
- configure apt to retry on network failures
- avoid running empty test splits
- remove per-agent throttle (as they are now dockerised)
patch by Mick Semb Wever; reviewed by Tomek Łasica for CASSANDRA-16604
---
build-scripts/cassandra-dtest-pytest-docker.sh | 2 +
build-scripts/cassandra-test-docker.sh | 132 ++++++++++++++++++-------
build-scripts/cassandra-test.sh | 62 ++++++++----
docker/buster-image.docker | 3 +
docker/centos8-image.docker | 2 +-
docker/jessie-image.docker | 3 +
docker/testing/ubuntu1910_j11.docker | 3 +
docker/testing/ubuntu2004_j11.docker | 3 +
jenkins-dsl/cassandra_job_dsl_seed.groovy | 34 ++++---
9 files changed, 178 insertions(+), 66 deletions(-)
diff --git a/build-scripts/cassandra-dtest-pytest-docker.sh b/build-scripts/cassandra-dtest-pytest-docker.sh
old mode 100644
new mode 100755
index 1e2efe6..c1ed7a0
--- a/build-scripts/cassandra-dtest-pytest-docker.sh
+++ b/build-scripts/cassandra-dtest-pytest-docker.sh
@@ -45,6 +45,7 @@ EOF
# docker login to avoid rate-limiting apache images. credentials are expected to already be in place
docker login || true
+ [[ "$(docker images -q $DOCKER_IMAGE 2>/dev/null)" != "" ]] || docker pull -q $DOCKER_IMAGE
echo "cassandra-dtest-pytest-docker.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET $SPLIT_CHUNK"
ID=$(docker run -m 15g --memory-swap 15g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET $SPLIT_CHUNK")
@@ -76,4 +77,5 @@ EOF
fi
docker rm $ID
+ exit $status
fi
\ No newline at end of file
diff --git a/build-scripts/cassandra-test-docker.sh b/build-scripts/cassandra-test-docker.sh
old mode 100644
new mode 100755
index c2e0973..4d2cd6a
--- a/build-scripts/cassandra-test-docker.sh
+++ b/build-scripts/cassandra-test-docker.sh
@@ -1,7 +1,7 @@
#!/bin/bash
#
# A wrapper script to cassandra-test.sh
-# that runs it in docker, collecting results.
+# that split the test list into multiple docker runs, collecting results.
#
# The docker image used is normally based from those found in docker/testing/
#
@@ -22,14 +22,19 @@ if [ "$#" -lt 3 ]; then
fi
java -version
javac -version
- echo "running: git clone --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git"
- git clone --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git
+ echo "running: git clone --quiet --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git"
+ until git clone --quiet --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git ; do echo "git clone failed… trying again… " ; done
cd cassandra
echo "cassandra-test.sh (${1} ${2}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" | tee "${1}-$(echo $2 | sed 's/\//-/')-cassandra.head"
echo "cassandra-test.sh (${1} ${2}) cassandra-builds: `git -C ../cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" | tee -a "${1}-$(echo $2 | sed 's/\//-/')-cassandra.head"
- bash ../cassandra-builds/build-scripts/cassandra-test.sh "$@"
+ ../cassandra-builds/build-scripts/cassandra-test.sh "$@"
if [ -d build/test/logs ]; then find build/test/logs -type f -name "*.log" | xargs xz -qq ; fi
else
+
+ # pre-conditions
+ command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; }
+ (docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; }
+
# start the docker container
if [ "$#" -lt 5 ]; then
echo "Usage: cassandra-test-docker.sh REPO BRANCH BUILDS_REPO_URL BUILDS_BRANCH DOCKER_IMAGE [target] [split_chunk]"
@@ -38,8 +43,8 @@ else
BUILDSREPO=$3
BUILDSBRANCH=$4
DOCKER_IMAGE=$5
- TARGET=$6
- SPLIT_CHUNK=$7
+ TARGET=${6:-"test"}
+ SPLIT_CHUNK=${7:-"1/1"}
# Setup JDK
java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
@@ -59,37 +64,98 @@ BRANCH=$2
JAVA_VERSION=${java_version}
EOF
+ # for relevant test targets calculate how many docker containers we should split the test list over
+ case $TARGET in
+ # test-burn doesn't have enough tests in it to split beyond 8
+ "stress-test" | "fqltool-test" | "microbench" | "test-burn")
+ docker_runs=1
+ ;;
+ "test"| "test-cdc" | "test-compression" | "long-test" | "jvm-dtest" | "jvm-dtest-upgrade")
+ cores=1
+ cores=$(nproc --all)
+ mem=1
+ # linux
+ command -v free >/dev/null 2>&1 && mem=$(free -b | grep Mem: | awk '{print $2}')
+ # macos
+ sysctl -n hw.memsize >/dev/null 2>&1 && mem=$(sysctl -n hw.memsize)
+ # Jenkins agents run multiple executors per machine. `jenkins_executors=1` is used for anything non-jenkins.
+ jenkins_executors=1
+ if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then
+ jenkins_executors=$(curl -s "${JENKINS_URL}/computer/${NODE_NAME}/api/json?pretty=true" | grep 'numExecutors' | awk -F' : ' '{print $2}' | cut -d',' -f1)
+ fi
+ max_docker_runs_by_cores=$( echo "sqrt( $cores / $jenkins_executors )" | bc )
+ max_docker_runs_by_mem=$(( $mem / ( 5 * 1024 * 1024 * 1024 * $jenkins_executors ) ))
+ docker_runs=$(( $max_docker_runs_by_cores < $max_docker_runs_by_mem ? $max_docker_runs_by_cores : $max_docker_runs_by_mem ))
+ docker_runs=$(( $docker_runs < 1 ? 1 : $docker_runs ))
+ ;;
+ *)
+ echo "unregconized \"$target\""
+ exit 1
+ ;;
+ esac
+
+ # Break up the requested split chunk into a number of concurrent docker runs, as calculated above
+ # This will typically be between one to four splits. Five splits would require >25 cores and >25GB ram
+ INNER_SPLITS=$(( $(echo $SPLIT_CHUNK | cut -d"/" -f2 ) * $docker_runs ))
+ INNER_SPLIT_FIRST=$(( ( $(echo $SPLIT_CHUNK | cut -d"/" -f1 ) * $docker_runs ) - ( $docker_runs - 1 ) ))
+
# docker login to avoid rate-limiting apache images. credentials are expected to already be in place
docker login || true
+ [[ "$(docker images -q $DOCKER_IMAGE 2>/dev/null)" != "" ]] || docker pull -q $DOCKER_IMAGE
- echo "cassandra-test-docker.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET $SPLIT_CHUNK"
- ID=$(docker run -m 15g --memory-swap 15g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET $SPLIT_CHUNK")
+ mkdir -p build/test/logs
+ declare -a DOCKER_IDS
+ declare -a PROCESS_IDS
+ declare -a STATUSES
- # use docker attach instead of docker wait to get output
- docker attach --no-stdin $ID
- status="$?"
+ for i in `seq 1 $docker_runs` ; do
+ inner_split=$(( $INNER_SPLIT_FIRST + ( $i - 1 ) ))
+ # start the container
+ echo "cassandra-test-docker.sh: running: git clone --quiet --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; bash ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET ${inner_split}/${INNER_SPLITS}"
+ docker_id=$(docker run -m 5g --memory-swap 5g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "until git clone --quiet --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO ; do echo 'git clone failed… trying again… ' ; done ; ./cassandra-builds/build-scripts/cassandra-test-docker.sh ${TARGET} ${inner_split}/${INNER_SPLITS}")
- if [ "$status" -ne 0 ] ; then
- echo "$ID failed (${status}), debug…"
- docker inspect $ID
- echo "–––"
- docker logs $ID
- echo "–––"
- docker ps -a
- echo "–––"
- docker info
- echo "–––"
- dmesg
- else
- echo "$ID done (${status}), copying files"
- # test meta
- docker cp "$ID:/home/cassandra/cassandra/${TARGET}-$(echo $SPLIT_CHUNK | sed 's/\//-/')-cassandra.head" .
- # test results
- mkdir -p build/test
- docker cp $ID:/home/cassandra/cassandra/build/test/output/. build/test/output
- # test logs
- docker cp $ID:/home/cassandra/cassandra/build/test/logs/. build/test/logs
- fi
+ # capture logs and pid for container
+ docker attach --no-stdin $docker_id > build/test/logs/docker_attach_${i}.log &
+ PROCESS_IDS+=( $! )
+ DOCKER_IDS+=( $docker_id )
+ done
+
+ exit_result=0
+ i=0
+ for process_id in "${PROCESS_IDS[@]}" ; do
+ # wait for each container to complete
+ docker_id=${DOCKER_IDS[$i]}
+ inner_split=$(( $INNER_SPLIT_FIRST + $i ))
+ cat build/test/logs/docker_attach_$(( $i + 1 )).log
+ tail -F build/test/logs/docker_attach_$(( $i + 1 )).log &
+ tail_process=$!
+ wait $process_id
+ status=$?
+ PROCESS_IDS+=( $status )
+ kill $tail_process
+
+ if [ "$status" -ne 0 ] ; then
+ echo "${docker_id} failed (${status}), debug…"
+ docker inspect ${docker_id}
+ echo "–––"
+ docker logs ${docker_id}
+ echo "–––"
+ docker ps -a
+ echo "–––"
+ docker info
+ echo "–––"
+ dmesg
+ exit_result=1
+ else
+ echo "${docker_id} done (${status}), copying files"
+ docker cp "$docker_id:/home/cassandra/cassandra/${TARGET}-${inner_split}-${INNER_SPLITS}-cassandra.head" .
+ docker cp $docker_id:/home/cassandra/cassandra/build/test/output/. build/test/output
+ docker cp $docker_id:/home/cassandra/cassandra/build/test/logs/. build/test/logs
+ fi
+ docker rm $docker_id
+ ((i++))
+ done
- docker rm $ID
+ xz build/test/logs/docker_attach_*.log
+ exit $exit_result
fi
diff --git a/build-scripts/cassandra-test.sh b/build-scripts/cassandra-test.sh
index ae84f4c..0bb428d 100755
--- a/build-scripts/cassandra-test.sh
+++ b/build-scripts/cassandra-test.sh
@@ -5,6 +5,11 @@ set -o errexit
set -o pipefail
set -o nounset
+# pre-conditions
+command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; }
+command -v git >/dev/null 2>&1 || { echo >&2 "git needs to be installed"; exit 1; }
+[ -f "build.xml" ] || { echo >&2 "build.xml must exist"; exit 1; }
+
# lists all tests for the specific test type
_list_tests() {
local -r classlistprefix="$1"
@@ -26,7 +31,7 @@ _timeout_for() {
_build_all_dtest_jars() {
cd $TMP_DIR
- git clone --depth 1 --no-single-branch https://gitbox.apache.org/repos/asf/cassandra.git cassandra-dtest-jars
+ until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
cd cassandra-dtest-jars
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 trunk; do
git checkout $branch
@@ -36,10 +41,24 @@ _build_all_dtest_jars() {
done
cd ../..
rm -fR ${TMP_DIR}/cassandra-dtest-jars
- ant dtest-jar
+ ant clean dtest-jar
ls -l build/dtest*.jar
}
+_run_testlist() {
+ local _target_prefix=$1
+ local _testlist_target=$2
+ local _split_chunk=$3
+ local _test_timeout=$4
+ testlist="$( _list_tests "${_target_prefix}" | _split_tests "${_split_chunk}")"
+ if ! [[ -z "$testlist" ]]; then
+ ant clean jar
+ ant $_testlist_target -Dtest.classlistprefix="${_target_prefix}" -Dtest.classlistfile=<(echo "${testlist}") -Dtest.timeout="${_test_timeout}" -Dtmp.dir="${TMP_DIR}" || echo "failed ${_target_prefix} ${$_testlist_target}"
+ else
+ echo Skipping ${_target_prefix} ${_testlist_target}, no tests in split ${_split_chunk}
+ fi
+}
+
_main() {
# parameters
local -r target="${1:-}"
@@ -74,50 +93,53 @@ _main() {
export TMP_DIR="$(pwd)/tmp"
mkdir -p ${TMP_DIR}
- ant clean jar
case $target in
"stress-test")
- # hard fail on test compilation, put dont fail the test run as unstable test reports are processed
- ant stress-build-test
+ # hard fail on test compilation, but dont fail the test run as unstable test reports are processed
+ ant clean stress-build-test jar
ant $target -Dtmp.dir="$(pwd)/tmp" || echo "failed $target"
;;
"fqltool-test")
- # hard fail on test compilation, put dont fail the test run so unstable test reports are processed
- ant fqltool-build-test
+ # hard fail on test compilation, but dont fail the test run so unstable test reports are processed
+ ant clean fqltool-build-test jar
ant $target -Dtmp.dir="$(pwd)/tmp" || echo "failed $target"
;;
"microbench")
- ant $target -Dtmp.dir="$(pwd)/tmp" -Dmaven.test.failure.ignore=true
+ ant clean $target -Dtmp.dir="$(pwd)/tmp" -Dmaven.test.failure.ignore=true
;;
"test")
- testlist="$( _list_tests "unit" | _split_tests "${split_chunk}")"
- ant testclasslist -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ _run_testlist "unit" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.timeout')"
;;
"test-cdc")
- testlist=$( _list_tests "unit" | _split_tests "${split_chunk}")
- ant testclasslist-cdc -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ _run_testlist "unit" "testclasslist-cdc" "${split_chunk}" "$(_timeout_for 'test.timeout')"
;;
"test-compression")
- testlist=$( _list_tests "unit" | _split_tests "${split_chunk}")
- ant testclasslist-compression -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ _run_testlist "unit" "testclasslist-compression" "${split_chunk}" "$(_timeout_for 'test.timeout')"
;;
"test-burn")
- testlist=$( _list_tests "burn" | _split_tests "${split_chunk}")
- ant testclasslist -Dtest.classlistprefix=burn -Dtest.timeout=$(_timeout_for "test.burn.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ _run_testlist "burn" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.burn.timeout')"
;;
"long-test")
- testlist=$( _list_tests "long" | _split_tests "${split_chunk}")
- ant testclasslist -Dtest.classlistprefix=long -Dtest.timeout=$(_timeout_for "test.long.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ _run_testlist "long" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.long.timeout')"
;;
"jvm-dtest")
+ ant clean jar
testlist=$( _list_tests "distributed" | grep -v "upgrade" | _split_tests "${split_chunk}")
- ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ if ! [[ -z "$testlist" ]]; then
+ ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ else
+ echo "Skipping $target, no tests in split ${split_chunk}"
+ fi
;;
"jvm-dtest-upgrade")
_build_all_dtest_jars
testlist=$( _list_tests "distributed" | grep "upgrade" | _split_tests "${split_chunk}")
- ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target"
+ if ! [[ -z "$testlist" ]]; then
+ ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target"
+ else
+ echo "Skipping $target, no tests in split ${split_chunk}"
+ fi
;;
*)
echo "unregconised \"$target\""
diff --git a/docker/buster-image.docker b/docker/buster-image.docker
index 5139a09..4f554cd 100644
--- a/docker/buster-image.docker
+++ b/docker/buster-image.docker
@@ -8,6 +8,9 @@ LABEL org.cassandra.buildenv=buster
VOLUME ${DEB_DIST_DIR}
+# configure apt to retry downloads
+RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries
+
# install deps
RUN apt-get update && apt-get -y install \
ant \
diff --git a/docker/centos8-image.docker b/docker/centos8-image.docker
index 58f6565..0c7e0c3 100644
--- a/docker/centos8-image.docker
+++ b/docker/centos8-image.docker
@@ -20,7 +20,7 @@ RUN yum -y install \
sudo \
python3-pip
-RUN curl http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm
+RUN curl -S -s --retry 9 http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm
RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm
diff --git a/docker/jessie-image.docker b/docker/jessie-image.docker
index f3a8b69..d68e632 100644
--- a/docker/jessie-image.docker
+++ b/docker/jessie-image.docker
@@ -8,6 +8,9 @@ LABEL org.cassandra.buildenv=jessie
VOLUME ${DEB_DIST_DIR}
+# configure apt to retry downloads
+RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries
+
# install deps
RUN apt-get update && apt-get -y install \
ant \
diff --git a/docker/testing/ubuntu1910_j11.docker b/docker/testing/ubuntu1910_j11.docker
index 8226d9f..0449852 100644
--- a/docker/testing/ubuntu1910_j11.docker
+++ b/docker/testing/ubuntu1910_j11.docker
@@ -13,6 +13,9 @@
FROM ubuntu:19.10
MAINTAINER Apache Cassandra <de...@cassandra.apache.org>
+# configure apt to retry downloads
+RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries
+
# install our python dependencies and some other stuff we need
# libev4 libev-dev are for the python driver / libssl-dev is for python3.6
diff --git a/docker/testing/ubuntu2004_j11.docker b/docker/testing/ubuntu2004_j11.docker
index d442d21..48bc812 100644
--- a/docker/testing/ubuntu2004_j11.docker
+++ b/docker/testing/ubuntu2004_j11.docker
@@ -13,6 +13,9 @@
FROM ubuntu:20.04
MAINTAINER Apache Cassandra <de...@cassandra.apache.org>
+# configure apt to retry downloads
+RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries
+
# install our python dependencies and some other stuff we need
# libev4 libev-dev are for the python driver / libssl-dev is for python3.6
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 255f390..444bf59 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -186,9 +186,6 @@ matrixJob('Cassandra-template-test') {
}
timestamps()
}
- throttleConcurrentBuilds {
- categories(['Cassandra'])
- }
properties {
githubProjectUrl(githubRepo)
priorityJobProperty {
@@ -456,8 +453,16 @@ cassandraBranches.each {
node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
}
steps {
+ if (arch == "-arm64") {
+ shell("""
+ # docker image has to be built on arm64 (they are not currently published to dockerhub)
+ cd cassandra-builds/docker/testing ;
+ docker build -t ${dtestDockerImage}:latest -f ubuntu2004_j11.docker . ;
+ docker build -t ${testDockerImage}:latest -f ubuntu2004_j11_w_dependencies.docker .
+ """)
+ }
shell("""
- sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh apache ${branchName} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ;
+ ./cassandra-builds/build-scripts/cassandra-test-docker.sh apache ${branchName} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ;
./cassandra-builds/build-scripts/cassandra-test-report.sh ;
xz TESTS-TestSuites.xml
""")
@@ -555,7 +560,7 @@ cassandraBranches.each {
""")
}
shell("""
- sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git trunk ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits} ;
+ ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git trunk ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits} ;
""")
}
publishers {
@@ -821,9 +826,6 @@ testTargets.each {
}
timestamps()
}
- throttleConcurrentBuilds {
- categories(['Cassandra'])
- }
parameters {
stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from')
stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout')
@@ -859,10 +861,18 @@ testTargets.each {
git clean -xdff ${targetName == 'microbench' ? '-e build/test/jmh-result.json' : ''};
git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ;
echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ;
- echo "Cassandra-devbranch-${targetName}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head
- """)
+ echo "Cassandra-devbranch-${targetName}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head
+ """)
+ if (arch == "-arm64") {
+ shell("""
+ # docker image has to be built on arm64 (they are not currently published to dockerhub)
+ cd cassandra-builds/docker/testing ;
+ docker build -t ${dtestDockerImage}:latest -f ubuntu2004_j11.docker . ;
+ docker build -t ${testDockerImage}:latest -f ubuntu2004_j11_w_dependencies.docker .
+ """)
+ }
shell("""
- sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh \${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ;
+ ./cassandra-builds/build-scripts/cassandra-test-docker.sh \${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ;
./cassandra-builds/build-scripts/cassandra-test-report.sh ;
xz TESTS-TestSuites.xml
""")
@@ -1001,7 +1011,7 @@ archs.each {
""")
}
shell("""
- sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ;
+ ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ;
""")
}
publishers {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org