You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2021/05/27 15:40:44 UTC
[cassandra] branch cassandra-3.0 updated: Add CircleCI jobs to
repeat upgrade tests n times
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push:
new 269bc5c Add CircleCI jobs to repeat upgrade tests n times
269bc5c is described below
commit 269bc5c35b91e531cdacef7109e39d87e0bf9b3b
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Thu May 27 16:30:21 2021 +0100
Add CircleCI jobs to repeat upgrade tests n times
patch by Andrés de la Peña; reviewed by Ekaterina Dimitrova for CASSANDRA-16687
---
.circleci/config-2_1.yml | 178 ++++++--
.circleci/config.yml | 954 ++++++++++++++++++++++++++++++-------------
.circleci/config.yml.HIGHRES | 954 ++++++++++++++++++++++++++++++-------------
.circleci/config.yml.LOWRES | 954 ++++++++++++++++++++++++++++++-------------
4 files changed, 2169 insertions(+), 871 deletions(-)
diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml
index a04a370..03e4ae9 100644
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@ -34,6 +34,7 @@ default_env_vars: &default_env_vars
DTEST_BRANCH: trunk
CCM_MAX_HEAP_SIZE: 1024M
CCM_HEAP_NEWSIZE: 256M
+
# The Ant test target to run, for example:
# REPEATED_UTEST_TARGET: testsome
# REPEATED_UTEST_TARGET: test-jvm-dtest-some
@@ -54,6 +55,7 @@ default_env_vars: &default_env_vars
REPEATED_UTEST_COUNT: 100
# Whether the test iteration should stop on the first failure
REPEATED_UTEST_STOP_ON_FAILURE: false
+
# A Python dtest to be run multiple times, for example:
# REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py
# REPEATED_DTEST_NAME: cqlsh_tests/test_cqlsh.py::TestCqlshSmoke
@@ -66,6 +68,27 @@ default_env_vars: &default_env_vars
# Whether the test iteration should stop on the first failure
REPEATED_DTEST_STOP_ON_FAILURE: false
+ # A Python upgrade dtest to be run multiple times, for example:
+ # REPEATED_UPGRADE_DTEST_NAME: upgrade_tests/cql_tests.py
+ # REPEATED_UPGRADE_DTEST_NAME: upgrade_tests/cql_tests.py::TestCQLNodes2RF1_Upgrade_current_4_0_x_To_indev_4_0_x
+ REPEATED_UPGRADE_DTEST_NAME:
+ # The number of times that the repeated Python upgrade dtest should be run
+ REPEATED_UPGRADE_DTEST_COUNT: 100
+ # Whether the test iteration should stop on the first failure
+ REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+
+ # The name of JVM upgrade dtest class to be run multiple times, for example:
+ # REPEATED_JVM_UPGRADE_DTEST_CLASS: org.apache.cassandra.distributed.upgrade.MixedModeAvailabilityV30Test
+ REPEATED_JVM_UPGRADE_DTEST_CLASS:
+ # The optional specific methods within REPEATED_JVM_UPGRADE_DTEST_CLASS to be run, for example:
+ # REPEATED_JVM_UPGRADE_DTEST_METHODS: testAvailabilityV30ToV3X
+ # REPEATED_JVM_UPGRADE_DTEST_METHODS: testAvailabilityV30ToV3X,testAvailabilityV30ToV4
+ REPEATED_JVM_UPGRADE_DTEST_METHODS:
+ # The number of times that the repeated JVM upgrade dtest should be run
+ REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ # Whether the JVM upgrade dtest iteration should stop on the first failure
+ REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@ -115,14 +138,17 @@ with_dtests_jobs: &with_dtest_jobs
- utests_compression:
requires:
- start_utests_compression
- - start_jvm_upgrade_dtest:
+ - start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
requires:
- build
- - start_jvm_upgrade_dtest
+ - start_j8_dtest_jars_build
+ - start_jvm_upgrade_dtest:
+ type: approval
- j8_jvm_upgrade_dtests:
requires:
+ - start_jvm_upgrade_dtest
- j8_dtest_jars_build
# Java 8 dtests (on request)
- start_j8_dtests:
@@ -144,19 +170,33 @@ with_dtests_jobs: &with_dtest_jobs
requires:
- start_upgrade_tests
# Java 8 repeated utest (on request)
- - start_j8_repeated-utest:
+ - start_j8_repeated_utest:
type: approval
- - j8_repeated-utest:
+ - j8_repeated_utest:
requires:
- - start_j8_repeated-utest
+ - start_j8_repeated_utest
- build
# Java 8 repeated dtest (on request)
- - start_j8_repeated-dtest:
+ - start_j8_repeated_dtest:
type: approval
- - j8_repeated-dtest:
+ - j8_repeated_dtest:
requires:
- - start_j8_repeated-dtest
+ - start_j8_repeated_dtest
- build
+ # Repeated Python upgrade dtest (on request)
+ - start_repeated_upgrade_dtest:
+ type: approval
+ - repeated_upgrade_dtest:
+ requires:
+ - start_repeated_upgrade_dtest
+ - build
+ # Repeated JVM upgrade dtest (on request)
+ - start_repeated_jvm_upgrade_dtest:
+ type: approval
+ - repeated_jvm_upgrade_dtest:
+ requires:
+ - start_repeated_jvm_upgrade_dtest
+ - j8_dtest_jars_build
with_dtest_jobs_only: &with_dtest_jobs_only
jobs:
@@ -312,22 +352,59 @@ jobs:
extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
pytest_extra_args: '--execute-upgrade-tests'
- j8_repeated-utest:
+ j8_repeated_utest:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- - run_repeated_utest
-
- j8_repeated-dtest:
+ - run_repeated_utest:
+ target: ${REPEATED_UTEST_TARGET}
+ class: ${REPEATED_UTEST_CLASS}
+ methods: ${REPEATED_UTEST_METHODS}
+ count: ${REPEATED_UTEST_COUNT}
+ stop_on_failure: ${REPEATED_UTEST_STOP_ON_FAILURE}
+
+ j8_repeated_dtest:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - clone_dtest
+ - create_venv
+ - run_repeated_dtest:
+ tests: ${REPEATED_DTEST_NAME}
+ vnodes: ${REPEATED_DTEST_VNODES}
+ upgrade: "false"
+ count: ${REPEATED_DTEST_COUNT}
+ stop_on_failure: ${REPEATED_DTEST_STOP_ON_FAILURE}
+
+ repeated_jvm_upgrade_dtest:
+ <<: *j8_par_executor
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - log_environment
+ - run_repeated_utest:
+ target: test-jvm-dtest-some
+ class: ${REPEATED_JVM_UPGRADE_DTEST_CLASS}
+ methods: ${REPEATED_JVM_UPGRADE_DTEST_METHODS}
+ count: ${REPEATED_JVM_UPGRADE_DTEST_COUNT}
+ stop_on_failure: ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE}
+
+ repeated_upgrade_dtest:
<<: *j8_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- clone_dtest
- create_venv
- - run_repeated_dtest
+ - run_repeated_dtest:
+ tests: ${REPEATED_UPGRADE_DTEST_NAME}
+ vnodes: "false"
+ upgrade: "true"
+ stop_on_failure: ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}
+ count: ${REPEATED_UPGRADE_DTEST_COUNT}
commands:
log_environment:
@@ -648,30 +725,41 @@ commands:
destination: dtest_<<parameters.file_tag>>_logs
run_repeated_utest:
+ parameters:
+ target:
+ type: string
+ class:
+ type: string
+ methods:
+ type: string
+ count:
+ type: string
+ stop_on_failure:
+ type: string
steps:
- run:
- name: Run repeated utest
+ name: Run repeated JUnit test
no_output_timeout: 15m
command: |
- if [ "$REPEATED_UTEST_CLASS" == "<nil>" ]; then
+ if [ "<<parameters.class>>" == "<nil>" ]; then
echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" == "<nil>" ]; then
+ elif [ "<<parameters.count>>" == "<nil>" ]; then
echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then
+ elif [ "<<parameters.count>>" -le 0 ]; then
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then
+ count=$((<<parameters.count>> / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (<<parameters.count>> % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
- echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times"
+ echo "Running <<parameters.target>> <<parameters.class>> <<parameters.methods>> <<parameters.count>> times"
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -681,8 +769,8 @@ commands:
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- target=$REPEATED_UTEST_TARGET
- class_path=$REPEATED_UTEST_CLASS
+ target=<<parameters.target>>
+ class_path=<<parameters.class>>
class_name="${class_path##*.}"
# Prepare the -Dtest.name argument.
@@ -697,14 +785,14 @@ commands:
fi
# Prepare the -Dtest.methods argument, which is optional
- if [ "$REPEATED_UTEST_METHODS" == "<nil>" ]; then
+ if [ "<<parameters.methods>>" == "<nil>" ]; then
methods=""
else
- methods="-Dtest.methods=$REPEATED_UTEST_METHODS"
+ methods="-Dtest.methods=<<parameters.methods>>"
fi
# Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set.
+ # stopping the iteration only if stop_on_failure is set.
exit_code="$?"
for i in $(seq -w 1 $count); do
@@ -720,7 +808,7 @@ commands:
# move the stdout output file
dest=/tmp/results/repeated_utest/stdout/${status}/${i}
mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_UTEST_TARGET}-${REPEATED_UTEST_CLASS}.txt
+ mv stdout.txt $dest/<<parameters.target>>-<<parameters.class>>.txt
# move the XML output files
source=build/test/output
@@ -739,7 +827,7 @@ commands:
fi
# maybe stop iterations on test failure
- if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then
+ if [[ <<parameters.stop_on_failure>> = true ]] && (( $exit_code > 0 )); then
break
fi
done
@@ -760,30 +848,41 @@ commands:
destination: logs
run_repeated_dtest:
+ parameters:
+ tests:
+ type: string
+ vnodes:
+ type: string
+ upgrade:
+ type: string
+ count:
+ type: string
+ stop_on_failure:
+ type: string
steps:
- run:
- name: Run repeated dtest
+ name: Run repeated Python dtest
no_output_timeout: 15m
command: |
- if [ "$REPEATED_DTEST_NAME" == "<nil>" ]; then
+ if [ "<<parameters.tests>>" == "<nil>" ]; then
echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" == "<nil>" ]; then
+ elif [ "<<parameters.count>>" == "<nil>" ]; then
echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then
+ elif [ "<<parameters.count>>" -le 0 ]; then
echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then
+ count=$((<<parameters.count>> / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (<<parameters.count>> % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
- echo "Running $REPEATED_DTEST_NAME $count times"
+ echo "Running <<parameters.tests>> $count times"
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
@@ -797,17 +896,22 @@ commands:
mkdir -p /tmp/results/dtests
stop_on_failure_arg=""
- if $REPEATED_UTEST_STOP_ON_FAILURE; then
+ if <<parameters.stop_on_failure>>; then
stop_on_failure_arg="-x"
fi
vnodes_args=""
- if $REPEATED_DTEST_VNODES; then
+ if <<parameters.vnodes>>; then
vnodes_args="--use-vnodes --num-tokens=16"
fi
+ upgrade_arg=""
+ if <<parameters.upgrade>>; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir <<parameters.tests>> | tee /tmp/dtest/stdout.txt
fi
fi
- store_test_results:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 95aafcb..c87c848 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -117,6 +117,427 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_jvm_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=test-jvm-dtest-some
+ class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_repeated_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if ${REPEATED_DTEST_VNODES}; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
@@ -207,6 +628,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests-no-vnodes:
@@ -274,6 +702,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_upgradetests-no-vnodes:
@@ -382,9 +817,124 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-dtest:
+ j8_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
@@ -402,70 +952,29 @@ jobs:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated dtest
- no_output_timeout: 15m
- command: |
- if [ "$REPEATED_DTEST_NAME" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running $REPEATED_DTEST_NAME $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- stop_on_failure_arg=""
- if $REPEATED_UTEST_STOP_ON_FAILURE; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if $REPEATED_DTEST_VNODES; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt
- fi
- fi
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -490,20 +999,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
+ j8_jvm_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
+ name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -513,11 +1029,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -553,11 +1069,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -591,49 +1107,44 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ utests_long:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Run Unit Tests (long-test)
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
- - run:
- name: Run dtests (j8_with_vnodes)
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean long-test
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -658,20 +1169,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ utests_compression:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine distributed Tests to Run
+ name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -681,11 +1199,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -712,7 +1230,7 @@ jobs:
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist-compression)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -721,11 +1239,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -759,9 +1277,16 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-utest:
+ j8_repeated_utest:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
@@ -794,28 +1319,28 @@ jobs:
which java
java -version
- run:
- name: Run repeated utest
+ name: Run repeated JUnit test
no_output_timeout: 15m
command: |
- if [ "$REPEATED_UTEST_CLASS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_CLASS}" == "<nil>" ]; then
echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" == "<nil>" ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" == "<nil>" ]; then
echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then
+ count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
- echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times"
+ echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times"
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -825,8 +1350,8 @@ jobs:
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- target=$REPEATED_UTEST_TARGET
- class_path=$REPEATED_UTEST_CLASS
+ target=${REPEATED_UTEST_TARGET}
+ class_path=${REPEATED_UTEST_CLASS}
class_name="${class_path##*.}"
# Prepare the -Dtest.name argument.
@@ -841,14 +1366,14 @@ jobs:
fi
# Prepare the -Dtest.methods argument, which is optional
- if [ "$REPEATED_UTEST_METHODS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_METHODS}" == "<nil>" ]; then
methods=""
else
- methods="-Dtest.methods=$REPEATED_UTEST_METHODS"
+ methods="-Dtest.methods=${REPEATED_UTEST_METHODS}"
fi
# Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set.
+ # stopping the iteration only if stop_on_failure is set.
exit_code="$?"
for i in $(seq -w 1 $count); do
@@ -883,7 +1408,7 @@ jobs:
fi
# maybe stop iterations on test failure
- if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then
+ if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
break
fi
done
@@ -926,162 +1451,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant clean long-test
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist-compression)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
@@ -1169,6 +1545,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
@@ -1196,14 +1579,17 @@ workflows:
- utests_compression:
requires:
- start_utests_compression
- - start_jvm_upgrade_dtest:
+ - start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
requires:
- build
- - start_jvm_upgrade_dtest
+ - start_j8_dtest_jars_build
+ - start_jvm_upgrade_dtest:
+ type: approval
- j8_jvm_upgrade_dtests:
requires:
+ - start_jvm_upgrade_dtest
- j8_dtest_jars_build
- start_j8_dtests:
type: approval
@@ -1222,15 +1608,27 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
- - start_j8_repeated-utest:
+ - start_j8_repeated_utest:
+ type: approval
+ - j8_repeated_utest:
+ requires:
+ - start_j8_repeated_utest
+ - build
+ - start_j8_repeated_dtest:
type: approval
- - j8_repeated-utest:
+ - j8_repeated_dtest:
requires:
- - start_j8_repeated-utest
+ - start_j8_repeated_dtest
- build
- - start_j8_repeated-dtest:
+ - start_repeated_upgrade_dtest:
type: approval
- - j8_repeated-dtest:
+ - repeated_upgrade_dtest:
requires:
- - start_j8_repeated-dtest
+ - start_repeated_upgrade_dtest
- build
+ - start_repeated_jvm_upgrade_dtest:
+ type: approval
+ - repeated_jvm_upgrade_dtest:
+ requires:
+ - start_repeated_jvm_upgrade_dtest
+ - j8_dtest_jars_build
diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES
index 7035295..521e6c0 100644
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@ -117,6 +117,427 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_jvm_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=test-jvm-dtest-some
+ class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_repeated_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if ${REPEATED_DTEST_VNODES}; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
@@ -207,6 +628,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests-no-vnodes:
@@ -274,6 +702,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_upgradetests-no-vnodes:
@@ -382,9 +817,124 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-dtest:
+ j8_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: xlarge
@@ -402,70 +952,29 @@ jobs:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated dtest
- no_output_timeout: 15m
- command: |
- if [ "$REPEATED_DTEST_NAME" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running $REPEATED_DTEST_NAME $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- stop_on_failure_arg=""
- if $REPEATED_UTEST_STOP_ON_FAILURE; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if $REPEATED_DTEST_VNODES; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt
- fi
- fi
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -490,20 +999,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
+ j8_jvm_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 100
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
+ name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -513,11 +1029,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -553,11 +1069,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -591,49 +1107,44 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ utests_long:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 100
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Run Unit Tests (long-test)
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
- - run:
- name: Run dtests (j8_with_vnodes)
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean long-test
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -658,20 +1169,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ utests_compression:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 100
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine distributed Tests to Run
+ name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -681,11 +1199,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -712,7 +1230,7 @@ jobs:
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist-compression)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -721,11 +1239,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -759,9 +1277,16 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-utest:
+ j8_repeated_utest:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: xlarge
@@ -794,28 +1319,28 @@ jobs:
which java
java -version
- run:
- name: Run repeated utest
+ name: Run repeated JUnit test
no_output_timeout: 15m
command: |
- if [ "$REPEATED_UTEST_CLASS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_CLASS}" == "<nil>" ]; then
echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" == "<nil>" ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" == "<nil>" ]; then
echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then
+ count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
- echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times"
+ echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times"
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -825,8 +1350,8 @@ jobs:
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- target=$REPEATED_UTEST_TARGET
- class_path=$REPEATED_UTEST_CLASS
+ target=${REPEATED_UTEST_TARGET}
+ class_path=${REPEATED_UTEST_CLASS}
class_name="${class_path##*.}"
# Prepare the -Dtest.name argument.
@@ -841,14 +1366,14 @@ jobs:
fi
# Prepare the -Dtest.methods argument, which is optional
- if [ "$REPEATED_UTEST_METHODS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_METHODS}" == "<nil>" ]; then
methods=""
else
- methods="-Dtest.methods=$REPEATED_UTEST_METHODS"
+ methods="-Dtest.methods=${REPEATED_UTEST_METHODS}"
fi
# Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set.
+ # stopping the iteration only if stop_on_failure is set.
exit_code="$?"
for i in $(seq -w 1 $count); do
@@ -883,7 +1408,7 @@ jobs:
fi
# maybe stop iterations on test failure
- if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then
+ if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
break
fi
done
@@ -926,162 +1451,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant clean long-test
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist-compression)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 2048M
- - CCM_HEAP_NEWSIZE: 512M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
@@ -1169,6 +1545,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
@@ -1196,14 +1579,17 @@ workflows:
- utests_compression:
requires:
- start_utests_compression
- - start_jvm_upgrade_dtest:
+ - start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
requires:
- build
- - start_jvm_upgrade_dtest
+ - start_j8_dtest_jars_build
+ - start_jvm_upgrade_dtest:
+ type: approval
- j8_jvm_upgrade_dtests:
requires:
+ - start_jvm_upgrade_dtest
- j8_dtest_jars_build
- start_j8_dtests:
type: approval
@@ -1222,15 +1608,27 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
- - start_j8_repeated-utest:
+ - start_j8_repeated_utest:
+ type: approval
+ - j8_repeated_utest:
+ requires:
+ - start_j8_repeated_utest
+ - build
+ - start_j8_repeated_dtest:
type: approval
- - j8_repeated-utest:
+ - j8_repeated_dtest:
requires:
- - start_j8_repeated-utest
+ - start_j8_repeated_dtest
- build
- - start_j8_repeated-dtest:
+ - start_repeated_upgrade_dtest:
type: approval
- - j8_repeated-dtest:
+ - repeated_upgrade_dtest:
requires:
- - start_j8_repeated-dtest
+ - start_repeated_upgrade_dtest
- build
+ - start_repeated_jvm_upgrade_dtest:
+ type: approval
+ - repeated_jvm_upgrade_dtest:
+ requires:
+ - start_repeated_jvm_upgrade_dtest
+ - j8_dtest_jars_build
diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES
index 95aafcb..c87c848 100644
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@ -117,6 +117,427 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_jvm_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_JVM_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_JVM_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running test-jvm-dtest-some ${REPEATED_JVM_UPGRADE_DTEST_CLASS} ${REPEATED_JVM_UPGRADE_DTEST_METHODS} ${REPEATED_JVM_UPGRADE_DTEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=test-jvm-dtest-some
+ class_path=${REPEATED_JVM_UPGRADE_DTEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_JVM_UPGRADE_DTEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_JVM_UPGRADE_DTEST_METHODS}"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/test-jvm-dtest-some-${REPEATED_JVM_UPGRADE_DTEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ repeated_upgrade_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_UPGRADE_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_repeated_dtest:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTEST_NAME}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTEST_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTEST_NAME} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ stop_on_failure_arg=""
+ if ${REPEATED_DTEST_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if ${REPEATED_DTEST_VNODES}; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir ${REPEATED_DTEST_NAME} | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
@@ -207,6 +628,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtests-no-vnodes:
@@ -274,6 +702,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_upgradetests-no-vnodes:
@@ -382,9 +817,124 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-dtest:
+ j8_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_UTEST_TARGET: testsome
+ - REPEATED_UTEST_CLASS: null
+ - REPEATED_UTEST_METHODS: null
+ - REPEATED_UTEST_COUNT: 100
+ - REPEATED_UTEST_STOP_ON_FAILURE: false
+ - REPEATED_DTEST_NAME: null
+ - REPEATED_DTEST_VNODES: false
+ - REPEATED_DTEST_COUNT: 100
+ - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
@@ -402,70 +952,29 @@ jobs:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Run repeated dtest
- no_output_timeout: 15m
- command: |
- if [ "$REPEATED_DTEST_NAME" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_DTEST_COUNT" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_DTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_DTEST_COUNT % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running $REPEATED_DTEST_NAME $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- stop_on_failure_arg=""
- if $REPEATED_UTEST_STOP_ON_FAILURE; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if $REPEATED_DTEST_VNODES; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $REPEATED_DTEST_NAME | tee /tmp/dtest/stdout.txt
- fi
- fi
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/dtest
- destination: dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
path: ~/cassandra-dtest/logs
- destination: dtest_logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -490,20 +999,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests:
+ j8_jvm_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
+ name: Determine distributed Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -513,11 +1029,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -553,11 +1069,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -591,49 +1107,44 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_dtests-with-vnodes:
+ utests_long:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 4
+ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Run Unit Tests (long-test)
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-r [...]
- - run:
- name: Run dtests (j8_with_vnodes)
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant clean long-test
no_output_timeout: 15m
- command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
@@ -658,20 +1169,27 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests:
+ utests_compression:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
- parallelism: 1
+ parallelism: 4
steps:
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine distributed Tests to Run
+ name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
@@ -681,11 +1199,11 @@ jobs:
echo "***java tests***"
# get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
@@ -712,7 +1230,7 @@ jobs:
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist-compression)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -721,11 +1239,11 @@ jobs:
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@ -759,9 +1277,16 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_repeated-utest:
+ j8_repeated_utest:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
resource_class: medium
@@ -794,28 +1319,28 @@ jobs:
which java
java -version
- run:
- name: Run repeated utest
+ name: Run repeated JUnit test
no_output_timeout: 15m
command: |
- if [ "$REPEATED_UTEST_CLASS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_CLASS}" == "<nil>" ]; then
echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" == "<nil>" ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" == "<nil>" ]; then
echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "$REPEATED_UTEST_COUNT" -le 0 ]; then
+ elif [ "${REPEATED_UTEST_COUNT}" -le 0 ]; then
echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
else
# Calculate the number of test iterations to be run by the current parallel runner.
# Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((REPEATED_UTEST_COUNT / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (REPEATED_UTEST_COUNT % CIRCLE_NODE_TOTAL))); then
+ count=$((${REPEATED_UTEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTEST_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
if (($count <= 0)); then
echo "No tests to run in this runner"
else
- echo "Running $REPEATED_UTEST_TARGET $REPEATED_UTEST_CLASS $REPEATED_UTEST_METHODS $count times"
+ echo "Running ${REPEATED_UTEST_TARGET} ${REPEATED_UTEST_CLASS} ${REPEATED_UTEST_METHODS} ${REPEATED_UTEST_COUNT} times"
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@ -825,8 +1350,8 @@ jobs:
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- target=$REPEATED_UTEST_TARGET
- class_path=$REPEATED_UTEST_CLASS
+ target=${REPEATED_UTEST_TARGET}
+ class_path=${REPEATED_UTEST_CLASS}
class_name="${class_path##*.}"
# Prepare the -Dtest.name argument.
@@ -841,14 +1366,14 @@ jobs:
fi
# Prepare the -Dtest.methods argument, which is optional
- if [ "$REPEATED_UTEST_METHODS" == "<nil>" ]; then
+ if [ "${REPEATED_UTEST_METHODS}" == "<nil>" ]; then
methods=""
else
- methods="-Dtest.methods=$REPEATED_UTEST_METHODS"
+ methods="-Dtest.methods=${REPEATED_UTEST_METHODS}"
fi
# Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if REPEATED_UTEST_STOP_ON_FAILURE is set.
+ # stopping the iteration only if stop_on_failure is set.
exit_code="$?"
for i in $(seq -w 1 $count); do
@@ -883,7 +1408,7 @@ jobs:
fi
# maybe stop iterations on test failure
- if [[ $REPEATED_UTEST_STOP_ON_FAILURE = true ]] && (( $exit_code > 0 )); then
+ if [[ ${REPEATED_UTEST_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
break
fi
done
@@ -926,162 +1451,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant clean long-test
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_compression:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:20210304
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Determine unit Tests to Run
- command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- no_output_timeout: 15m
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run Unit Tests (testclasslist-compression)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - ANT_HOME: /usr/share/ant
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_UTEST_TARGET: testsome
- - REPEATED_UTEST_CLASS: null
- - REPEATED_UTEST_METHODS: null
- - REPEATED_UTEST_COUNT: 100
- - REPEATED_UTEST_STOP_ON_FAILURE: false
- - REPEATED_DTEST_NAME: null
- - REPEATED_DTEST_VNODES: false
- - REPEATED_DTEST_COUNT: 100
- - REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_dtest_jars_build:
@@ -1169,6 +1545,13 @@ jobs:
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_UPGRADE_DTEST_NAME: null
+ - REPEATED_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
+ - REPEATED_JVM_UPGRADE_DTEST_CLASS: null
+ - REPEATED_JVM_UPGRADE_DTEST_METHODS: null
+ - REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
+ - REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
workflows:
@@ -1196,14 +1579,17 @@ workflows:
- utests_compression:
requires:
- start_utests_compression
- - start_jvm_upgrade_dtest:
+ - start_j8_dtest_jars_build:
type: approval
- j8_dtest_jars_build:
requires:
- build
- - start_jvm_upgrade_dtest
+ - start_j8_dtest_jars_build
+ - start_jvm_upgrade_dtest:
+ type: approval
- j8_jvm_upgrade_dtests:
requires:
+ - start_jvm_upgrade_dtest
- j8_dtest_jars_build
- start_j8_dtests:
type: approval
@@ -1222,15 +1608,27 @@ workflows:
- j8_upgradetests-no-vnodes:
requires:
- start_upgrade_tests
- - start_j8_repeated-utest:
+ - start_j8_repeated_utest:
+ type: approval
+ - j8_repeated_utest:
+ requires:
+ - start_j8_repeated_utest
+ - build
+ - start_j8_repeated_dtest:
type: approval
- - j8_repeated-utest:
+ - j8_repeated_dtest:
requires:
- - start_j8_repeated-utest
+ - start_j8_repeated_dtest
- build
- - start_j8_repeated-dtest:
+ - start_repeated_upgrade_dtest:
type: approval
- - j8_repeated-dtest:
+ - repeated_upgrade_dtest:
requires:
- - start_j8_repeated-dtest
+ - start_repeated_upgrade_dtest
- build
+ - start_repeated_jvm_upgrade_dtest:
+ type: approval
+ - repeated_jvm_upgrade_dtest:
+ requires:
+ - start_repeated_jvm_upgrade_dtest
+ - j8_dtest_jars_build
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org