You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2022/11/04 16:30:26 UTC
[cassandra] branch cassandra-4.0 updated (6885bf9724 -> 5e705a8492)
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a change to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from 6885bf9724 Merge branch 'cassandra-3.11' into cassandra-4.0
new 95400649d8 Improve behaviour of the Ant flag to skip test building, no-build-test
new bdd6606486 Merge branch 'cassandra-3.0' into cassandra-3.11
new 5e705a8492 Merge branch 'cassandra-3.11' into cassandra-4.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 4 +--
.circleci/config.yml | 34 ++++++++++----------
.circleci/config.yml.HIGHRES | 34 ++++++++++----------
.circleci/config.yml.LOWRES | 34 ++++++++++----------
.circleci/config.yml.MIDRES | 34 ++++++++++----------
build.xml | 74 +++++++++++++++++++++++++++-----------------
6 files changed, 115 insertions(+), 99 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.11' into cassandra-4.0
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 5e705a84927bd4de1020adec94b4dcc86bd23354
Merge: 6885bf9724 bdd6606486
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Fri Nov 4 16:26:08 2022 +0000
Merge branch 'cassandra-3.11' into cassandra-4.0
.circleci/config-2_1.yml | 4 +--
.circleci/config.yml | 34 ++++++++++----------
.circleci/config.yml.HIGHRES | 34 ++++++++++----------
.circleci/config.yml.LOWRES | 34 ++++++++++----------
.circleci/config.yml.MIDRES | 34 ++++++++++----------
build.xml | 74 +++++++++++++++++++++++++++-----------------
6 files changed, 115 insertions(+), 99 deletions(-)
diff --cc .circleci/config-2_1.yml
index 1e35368e22,04dec86f9c..ff20ec4a16
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@@ -1996,7 -856,7 +1996,7 @@@ commands
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant <<parameters.target>>
- ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> -Dno-build-test=true
++ ant <<parameters.target>> -Dno-build-test=true
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/build/test/output/
diff --cc .circleci/config.yml
index 3bd389e0a3,30046af3ca..e4feeed771
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -456,273 -528,123 +456,273 @@@ jobs
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
fi
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
fi
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant stress-test
++ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -1191,33 -1017,42 +1191,33 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
+ name: Run Unit Tests (testclasslist)
command: |
+ set -x
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar build-test
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -1277,4942 -1109,75 +1277,4942 @@@
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Run repeated Python dtest
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
no_output_timeout: 15m
command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-compression
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-system-keyspace-directory
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant fqltool-test
++ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant long-test
++ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=testsome
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant testsome $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant stress-test
++ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-cdc
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=fqltool-test
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-system-keyspace-directory
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=stress-test-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
count=$((count+1))
fi
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --ex [...]
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_jvm_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-jvm-dtest-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-jvm-dtest-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=long-testsome
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/pylib
+ ./cassandra-cqlsh-tests.sh ..
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-jvm-dtest-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
fi
- vnodes_args=""
- if true; then
- vnodes_args="--use-vnodes --num-tokens=16"
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
fi
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
fi
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -6481,93 -1422,22 +6481,93 @@@
break
fi
done
- done
- (exit ${exit_code})
+
+ (exit ${exit_code})
+ fi
+ fi
- store_test_results:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results/repeated_utest/output
- store_artifacts:
- path: /tmp/results/repeated_utests/stdout
+ path: /tmp/results/repeated_utest/stdout
destination: stdout
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/results/repeated_utest/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/results/repeated_utest/logs
destination: logs
environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant long-test
++ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -6911,7 -1712,7 +6911,7 @@@
if [ -z "$test_timeout" ]; then
test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
no_output_timeout: 15m
- store_test_results:
path: /tmp/cassandra/build/test/output/
@@@ -7365,28 -2106,140 +7365,28 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Run repeated JUnit test
- no_output_timeout: 15m
+ name: Run Unit Tests (fqltool-test)
command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- ant fqltool-test
++ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
diff --cc .circleci/config.yml.HIGHRES
index ad637f34e8,2f1e95a979..d2c0721ed1
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -456,273 -528,123 +456,273 @@@ jobs
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- fi
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
fi
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
fi
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant stress-test
++ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -1191,33 -1017,42 +1191,33 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
+ name: Run Unit Tests (testclasslist)
command: |
+ set -x
export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean jar build-test
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
no_output_timeout: 15m
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
environment:
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -2283,93 -1955,113 +2283,93 @@@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant fqltool-test
++ ant fqltool-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
- ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
@@@ -2457,328 -2128,40 +2457,328 @@@
which java
java -version
- run:
- name: Run repeated JUnit test
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant long-test
++ ant long-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=testsome
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
# Prepare the -Dtest.name argument.
# It can be the fully qualified class name or the short class name, depending on the target.
@@@ -2838,4196 -2220,26 +2838,4196 @@@
# maybe stop iterations on test failure
if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
break
- fi
- done
-
- (exit ${exit_code})
- fi
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant stress-test
++ ant stress-test -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-with-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-cdc
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=fqltool-test
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-system-keyspace-directory
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_stress_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=stress-test-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if true; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_upgradetests_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --ex [...]
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_jvm_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-jvm-dtest-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py38_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
+ - run:
+ name: Run dtests (j11_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
+
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.8
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=test-jvm-dtest-some
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ # Put manually specified tests and automatically detected tests together, removing duplicates
+ tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
+ echo "Tests to be repeated: ${tests}"
+
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=long-testsome
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
+ # Run each test class as many times as requested.
+ exit_code="$?"
+ for test in $tests; do
+
+ # Split class and method names from the test name
+ if [[ $test =~ "#" ]]; then
+ class=${test%"#"*}
+ method=${test#*"#"}
+ else
+ class=$test
+ method=""
+ fi
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
+ $target == "stress-test" ]]; then
+ name_arg="-Dtest.name=${class##*.}"
+ else
+ name_arg="-Dtest.name=$class"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [[ $method == "" ]]; then
+ methods_arg=""
+ else
+ methods_arg="-Dtest.methods=$method"
+ fi
+
+ for i in $(seq -w 1 $count); do
+ echo "Running test $test, iteration $i of $count"
+
+ # run the test
+ status="passes"
+ if !( set -o pipefail && \
+ ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
+ tee stdout.txt \
+ ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utests/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${test}.txt
+
+ # move the XML output files
+ source=build/test/output/${testtag}
+ dest=/tmp/results/repeated_utests/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs/${testtag}
+ dest=/tmp/results/repeated_utests/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+ done
+ (exit ${exit_code})
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_unit_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_jvm_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 5
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_build:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra Repository (via git)
+ command: |
+ git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
+ - run:
+ name: Build Cassandra
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
+ for x in $(seq 1 3); do
+ ${ANT_HOME}/bin/ant clean realclean jar
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ no_output_timeout: 15m
+ - run:
+ name: Run eclipse-warnings
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ cd ~/cassandra
+ ant eclipse-warnings
+ - persist_to_workspace:
+ root: /home/cassandra
+ paths:
+ - cassandra
+ - .m2
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh-dtests-py2-no-vnodes:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
+ export CQLSH_PYTHON=/usr/bin/python2.7
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_dtests_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the e [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlshlib_tests:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run cqlshlib Unit Tests
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra/pylib
+ ./cassandra-cqlsh-tests.sh ..
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/pylib
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_jvm_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
... 11938 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org