You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2022/11/03 11:47:21 UTC
[cassandra] branch cassandra-4.1 updated (b7ef579703 -> 1d1866b86d)
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a change to branch cassandra-4.1
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from b7ef579703 Merge branch 'cassandra-4.0' into cassandra-4.1
new 116ce3bc05 CircleCI: Add jobs for missing specialized unit tests
new 7df905a429 Merge branch 'cassandra-3.0' into cassandra-3.11
new 6152cb08db Merge branch 'cassandra-3.11' into cassandra-4.0
new 1d1866b86d Merge branch 'cassandra-4.0' into cassandra-4.1
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.circleci/config-2_1.yml | 541 ++++-
.circleci/config-2_1.yml.high_res.patch | 13 +-
.circleci/config-2_1.yml.mid_res.patch | 39 +-
.circleci/config.yml | 3315 +++++++++++++++++++--------
.circleci/config.yml.HIGHRES | 3709 +++++++++++++++++++++---------
.circleci/config.yml.LOWRES | 3703 +++++++++++++++++++++---------
.circleci/config.yml.MIDRES | 3713 ++++++++++++++++++++++---------
.circleci/generate.sh | 18 +-
8 files changed, 10865 insertions(+), 4186 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-4.0' into cassandra-4.1
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-4.1
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 1d1866b86d524bcaf10cd9d5f547c9ca57a184c6
Merge: b7ef579703 6152cb08db
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Thu Nov 3 11:43:48 2022 +0000
Merge branch 'cassandra-4.0' into cassandra-4.1
.circleci/config-2_1.yml | 541 ++++-
.circleci/config-2_1.yml.high_res.patch | 13 +-
.circleci/config-2_1.yml.mid_res.patch | 39 +-
.circleci/config.yml | 3315 +++++++++++++++++++--------
.circleci/config.yml.HIGHRES | 3709 +++++++++++++++++++++---------
.circleci/config.yml.LOWRES | 3703 +++++++++++++++++++++---------
.circleci/config.yml.MIDRES | 3713 ++++++++++++++++++++++---------
.circleci/generate.sh | 18 +-
8 files changed, 10865 insertions(+), 4186 deletions(-)
diff --cc .circleci/config-2_1.yml
index a3e865089e,1e35368e22..7c348937de
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@@ -812,12 -1004,79 +1035,73 @@@ j11_pre-commit_jobs: &j11_pre-commit_jo
- j11_cqlsh_dtests_py3_vnode:
requires:
- j11_build
- - j11_cqlsh_dtests_py38_vnode:
- requires:
- - j11_build
- - j11_cqlsh-dtests-py2-no-vnodes:
- requires:
- - j11_build
- - j11_cqlsh_dtests_py3:
+ - j11_cqlsh_dtests_py38:
requires:
- j11_build
- - j11_cqlsh_dtests_py38:
+ - j11_cqlsh_dtests_py38_vnode:
requires:
- j11_build
+ # specialized unit tests (all run on request)
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - j11_utests_long_repeat:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - j11_utests_cdc_repeat:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - j11_utests_compression_repeat:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - j11_utests_stress_repeat:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - j11_utests_fqltool_repeat:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
+ - j11_utests_system_keyspace_directory_repeat:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
workflows:
version: 2
@@@ -1945,14 -2277,18 +2357,25 @@@ commands
# Put manually specified tests and automatically detected tests together, removing duplicates
tests=$(echo <<parameters.tests>> | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
echo "Tests to be repeated: ${tests}"
-
+
+ # Prepare the JVM dtests vnodes argument, which is optional.
+ vnodes=<<parameters.vnodes>>
+ vnodes_args=""
+ if [ "$vnodes" = true ] ; then
+ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
+ fi
-
++
+ # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
+ target=<<parameters.target>>
+ testtag=""
+ if [[ $target == "test-cdc" ]]; then
+ testtag="cdc"
+ elif [[ $target == "test-compression" ]]; then
+ testtag="compression"
+ elif [[ $target == "test-system-keyspace-directory" ]]; then
+ testtag="system_keyspace_directory"
+ fi
+
# Run each test class as many times as requested.
exit_code="$?"
for test in $tests; do
diff --cc .circleci/config-2_1.yml.high_res.patch
index 8ec657fd83,c8b629b095..1d3d0ed6be
--- a/.circleci/config-2_1.yml.high_res.patch
+++ b/.circleci/config-2_1.yml.high_res.patch
@@@ -1,6 -1,6 +1,6 @@@
- --- config-2_1.yml 2022-10-15 12:44:53.878216263 +0100
- +++ config-2_1.yml.HIGHRES 2022-10-15 12:45:49.850511983 +0100
---- config-2_1.yml 2022-10-27 10:22:41.868009084 +0100
-+++ config-2_1.yml.HIGHRES 2022-10-27 10:23:05.374865893 +0100
-@@ -136,14 +136,14 @@
++--- config-2_1.yml 2022-10-25 12:28:18.966893212 +0100
+++++ config-2_1.yml.HIGHRES 2022-10-25 12:29:12.625678547 +0100
+@@ -148,14 +148,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@@ -19,7 -19,7 +19,7 @@@
j8_small_executor: &j8_small_executor
executor:
- @@ -166,62 +166,68 @@
-@@ -154,68 +154,74 @@
++@@ -166,68 +166,74 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
diff --cc .circleci/config-2_1.yml.mid_res.patch
index a410c9b111,9fe0c53a7f..debd91f000
--- a/.circleci/config-2_1.yml.mid_res.patch
+++ b/.circleci/config-2_1.yml.mid_res.patch
@@@ -1,6 -1,6 +1,6 @@@
- --- config-2_1.yml 2022-10-15 12:44:53.878216263 +0100
- +++ config-2_1.yml.MIDRES 2022-10-15 12:45:49.843904887 +0100
---- config-2_1.yml 2022-10-27 10:22:41.868009084 +0100
-+++ config-2_1.yml.MIDRES 2022-10-27 10:23:05.368183086 +0100
-@@ -136,14 +136,14 @@
++--- config-2_1.yml 2022-10-25 12:28:18.966893212 +0100
+++++ config-2_1.yml.MIDRES 2022-10-25 12:29:12.619201467 +0100
+@@ -148,14 +148,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
@@@ -68,7 -65,7 +68,7 @@@
j11_small_executor: &j11_small_executor
executor:
- @@ -193,35 +205,47 @@
-@@ -181,41 +193,53 @@
++@@ -193,41 +205,53 @@
#exec_resource_class: medium
parallelism: 1
@@@ -122,7 -126,7 +129,7 @@@
j8_separate_jobs: &j8_separate_jobs
jobs:
- @@ -1045,7 +1069,7 @@
-@@ -1331,7 +1355,7 @@
++@@ -1393,7 +1417,7 @@
target: testclasslist-system-keyspace-directory
j8_dtests_vnode:
@@@ -131,7 -135,7 +138,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1059,7 +1083,7 @@
-@@ -1345,7 +1369,7 @@
++@@ -1407,7 +1431,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_vnode:
@@@ -140,7 -144,7 +147,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1074,7 +1098,7 @@
-@@ -1360,7 +1384,7 @@
++@@ -1422,7 +1446,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j8_dtests:
@@@ -149,7 -153,7 +156,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1088,7 +1112,7 @@
-@@ -1374,7 +1398,7 @@
++@@ -1436,7 +1460,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_dtests:
@@@ -158,7 -162,7 +165,7 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1103,7 +1127,7 @@
-@@ -1389,7 +1413,7 @@
++@@ -1451,7 +1475,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_upgrade_dtests:
@@@ -167,16 -171,25 +174,16 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1117,7 +1141,7 @@
-@@ -1403,7 +1427,7 @@
++@@ -1465,7 +1489,7 @@
pytest_extra_args: '--execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all'
- j8_cqlsh-dtests-py2-with-vnodes:
-- <<: *j8_par_executor
-+ <<: *j8_large_par_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -1418,7 +1442,7 @@
- extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
-
j8_cqlsh_dtests_py3_vnode:
- <<: *j8_par_executor
+ <<: *j8_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1132,7 +1156,7 @@
-@@ -1433,7 +1457,7 @@
++@@ -1480,7 +1504,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38_vnode:
@@@ -185,16 -198,25 +192,16 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1150,7 +1174,7 @@
-@@ -1451,7 +1475,7 @@
++@@ -1498,7 +1522,7 @@
python_version: '3.8'
- j8_cqlsh-dtests-py2-no-vnodes:
-- <<: *j8_par_executor
-+ <<: *j8_large_par_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -1466,7 +1490,7 @@
- extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
-
j8_cqlsh_dtests_py3:
- <<: *j8_par_executor
+ <<: *j8_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1165,7 +1189,7 @@
-@@ -1481,7 +1505,7 @@
++@@ -1513,7 +1537,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j8_cqlsh_dtests_py38:
@@@ -203,16 -225,25 +210,16 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1183,7 +1207,7 @@
-@@ -1499,7 +1523,7 @@
++@@ -1531,7 +1555,7 @@
python_version: '3.8'
- j11_cqlsh-dtests-py2-with-vnodes:
-- <<: *j11_par_executor
-+ <<: *j11_large_par_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -1514,7 +1538,7 @@
- extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
-
j11_cqlsh_dtests_py3_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1198,7 +1222,7 @@
-@@ -1529,7 +1553,7 @@
++@@ -1546,7 +1570,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
@@@ -221,16 -252,25 +228,16 @@@
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1216,7 +1240,7 @@
-@@ -1547,7 +1571,7 @@
++@@ -1564,7 +1588,7 @@
python_version: '3.8'
- j11_cqlsh-dtests-py2-no-vnodes:
-- <<: *j11_par_executor
-+ <<: *j11_large_par_executor
- steps:
- - attach_workspace:
- at: /home/cassandra
-@@ -1562,7 +1586,7 @@
- extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
-
j11_cqlsh_dtests_py3:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
- @@ -1231,7 +1255,7 @@
-@@ -1577,7 +1601,7 @@
++@@ -1579,7 +1603,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
diff --cc .circleci/config.yml
index f7fb72d98f,3bd389e0a3..3e06f9b7f2
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -170,7 -167,111 +170,7 @@@ jobs
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=fqltool-test
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -220,10 -319,110 +220,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j8_cqlsh-dtests-py2-with-vnodes:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_dtests_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@@ -412,19 -627,28 +432,28 @@@
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -464,11 -686,11 +493,12 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests_vnode:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_stress:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
@@@ -637,28 -791,123 +602,19 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=stress-test-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utests/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -703,170 -949,157 +659,170 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
- j8_cqlsh_dtests_py3:
- j8_utests_compression_repeat:
++ j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
-- parallelism: 4
++ parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Determine distributed Tests to Run
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
- run:
- name: Configure virtualenv and python Dependencies
+ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
- run:
- name: Run dtests (j8_without_vnodes)
- name: Repeatedly run new or modifed JUnit tests
-- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist)
command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
+ set -x
export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.6
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
--
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
--
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-compression
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
++ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh_dtests_py38:
++ j8_utests_compression_repeat:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Log Environment Information
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run dtests (j11_without_vnodes)
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -905,12 -1136,10 +861,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j11_repeated_ant_test:
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
@@@ -943,123 -1191,28 +917,28 @@@
which java
java -version
- run:
- name: Run repeated JUnit test
- no_output_timeout: 15m
+ name: Run Unit Tests (testclasslist)
command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" || \
- $target == "test-simulator-dtest" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Prepare the JVM dtests vnodes argument, which is optional
- vnodes_args=""
- if ${REPEATED_ANT_TEST_VNODES}; then
- vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
- fi
-
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1311,9 -1455,1073 +1190,896 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
- j11_jvm_dtests_vnode:
+ j11_utests_compression_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-compression
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_repeated_ant_test:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
- $target == "stress-test" ]]; then
++ $target == "stress-test" || \
++ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
++ # Prepare the JVM dtests vnodes argument, which is optional
++ vnodes_args=""
++ if ${REPEATED_ANT_TEST_VNODES}; then
++ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
++ fi
++
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
++ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-system-keyspace-directory
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_utests_compression:
++ j11_jvm_dtests_vnode:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -1363,7 -2571,172 +2129,238 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
- name: Run Unit Tests (testclasslist-compression)
++ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_utests_long:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_simulator_dtests:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (long-test)
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Simulator Tests
+ command: |
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant long-test
- no_output_timeout: 15m
++ ant test-simulator-dtest
++ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_unit_tests_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist-compression)
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
@@@ -1372,19 -2745,113 +2369,19 @@@
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1424,14 -2889,12 +2421,13 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_simulator_dtests:
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_cqlsh-dtests-py2-no-vnodes:
++ j11_utests_long:
docker:
-- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -1440,50 -2903,108 +2436,169 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
- - run:
- name: Run dtests (j8_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ ant long-test
no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_unit_tests_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 4
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
+ - run:
+ name: Log Environment Information
command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - run:
- name: Run Simulator Tests
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
command: |
- set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
@@@ -1538,11 -3057,11 +2653,12 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
@@@ -1631,9 -3190,110 +2790,9 @@@
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests_repeat:
- j11_cqlsh-dtests-py2-with-vnodes:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
- - run:
- name: Run dtests (j11_with_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc_repeat:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -1652,21 -3312,125 +2811,21 @@@
free -m
echo '*** df -m ***'
df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-cdc
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -1716,13 -3478,13 +2875,14 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_repeat:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -1848,46 -3630,150 +3030,136 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_fqltool_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and [...]
++ - store_test_results:
++ path: /tmp/results/repeated_utests/output
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_system_keyspace_directory_repeat:
++ j8_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=fqltool-test
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -1973,6 -3875,80 +3264,83 @@@
echo '*** java ***'
which java
java -version
+ - run:
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py38:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
@@@ -1990,159 -3966,40 +3358,129 @@@
pip3 uninstall -y cqlsh
pip3 freeze
- run:
- name: Run repeated Python dtest
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
- name: Run dtests (j8_without_vnodes)
++ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
++ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.8
++ fi
+
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
+
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_jvm_dtests_vnode_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -2196,56 -4050,146 +3534,42 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
- run:
- name: Run dtests (j8_without_vnodes)
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
-- command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- set -x
-- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
-- fi
--
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
--
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-system-keyspace-directory
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -2323,7 -4264,111 +3647,7 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=stress-test-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -2698,10 -4645,10 +3932,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_upgrade_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -2782,46 -4727,149 +4017,46 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_upgrade_dtests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_jvm_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified t [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified t [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -3014,7 -5056,111 +4249,7 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -3080,23 -5223,145 +4315,41 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Run Unit Tests (stress-test)
+ name: Log Environment Information
command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant stress-test
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and autom [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utests/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -3408,20 -5662,39 +4661,20 @@@
which java
java -version
- run:
- name: Clone Cassandra Repository (via git)
- command: |
- git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH https://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- - run:
- name: Build Cassandra
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
- for x in $(seq 1 3); do
- ${ANT_HOME}/bin/ant clean realclean jar
- RETURN="$?"
- if [ "${RETURN}" -eq "0" ]; then
- break
- fi
- done
- # Exit, if we didn't build successfully
- if [ "${RETURN}" -ne "0" ]; then
- echo "Build failed with exit code: ${RETURN}"
- exit ${RETURN}
- fi
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- - run:
- name: Run eclipse-warnings
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- cd ~/cassandra
- ant eclipse-warnings
- - persist_to_workspace:
- root: /home/cassandra
- paths:
- - cassandra
- - .m2
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -3838,7 -6093,111 +5091,7 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-jvm-dtest-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -4130,25 -6476,207 +5383,213 @@@
mv $source/* $dest/
fi
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results/repeated_utest/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_long:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (long-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant long-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
- (exit ${exit_code})
- fi
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -4630,133 -7146,163 +6071,62 @@@
- REPEATED_UTESTS_FQLTOOL: null
- REPEATED_UTESTS_FQLTOOL_COUNT: 500
- REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_unit_tests_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 4
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: medium
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 1
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Run Unit Tests (long-test)
- command: |
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- ant long-test
- no_output_timeout: 15m
- - store_test_results:
- path: /tmp/cassandra/build/test/output/
- - store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
- - store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_SIMULATOR_DTESTS: null
- - REPEATED_SIMULATOR_DTESTS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_VNODES: false
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_unit_tests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -5179,24 -7596,195 +6432,94 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Determine unit Tests to Run
+ name: Log Environment Information
command: |
- # reminder: this code (along with all the steps) is independently executed on every circle container
- # so the goal here is to get the circleci script to return the tests *this* container will run
- # which we do via the `circleci` cli tool.
-
- rm -fr ~/cassandra-dtest/upgrade_tests
- echo "***java tests***"
-
- # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
-
- # split up the unit tests into groups based on the number of containers we have
- set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
- cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-cdc
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_long_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
- run:
name: Log Environment Information
command: |
@@@ -5220,28 -7808,123 +6543,19 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist-compression)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_LONG} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=long-testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant long-testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and autom [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utests/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -5281,10 -7962,10 +6595,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_dtest_jars_build:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -5739,9 -8539,51 +7164,45 @@@ workflows
- j11_cqlsh_dtests_py3_vnode:
requires:
- j11_build
- - j11_cqlsh_dtests_py38_vnode:
- requires:
- - j11_build
- - j11_cqlsh-dtests-py2-no-vnodes:
- requires:
- - j11_build
- - j11_cqlsh_dtests_py3:
+ - j11_cqlsh_dtests_py38:
requires:
- j11_build
- - j11_cqlsh_dtests_py38:
+ - j11_cqlsh_dtests_py38_vnode:
requires:
- j11_build
+ - start_utests_long:
+ type: approval
+ - j11_utests_long:
+ requires:
+ - start_utests_long
+ - j11_build
+ - start_utests_cdc:
+ type: approval
+ - j11_utests_cdc:
+ requires:
+ - start_utests_cdc
+ - j11_build
+ - start_utests_compression:
+ type: approval
+ - j11_utests_compression:
+ requires:
+ - start_utests_compression
+ - j11_build
+ - start_utests_stress:
+ type: approval
+ - j11_utests_stress:
+ requires:
+ - start_utests_stress
+ - j11_build
+ - start_utests_fqltool:
+ type: approval
+ - j11_utests_fqltool:
+ requires:
+ - start_utests_fqltool
+ - j11_build
+ - start_utests_system_keyspace_directory:
+ type: approval
+ - j11_utests_system_keyspace_directory:
+ requires:
+ - start_utests_system_keyspace_directory
+ - j11_build
diff --cc .circleci/config.yml.HIGHRES
index 7e5c9d1359,ad637f34e8..7c8c2ddc61
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -170,7 -167,111 +170,7 @@@ jobs
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=fqltool-test
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -220,10 -319,110 +220,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
- j8_cqlsh-dtests-py2-with-vnodes:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
- - run:
- name: Run dtests (j8_with_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j11_dtests_vnode_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
@@@ -412,19 -627,28 +432,28 @@@
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -464,11 -686,11 +493,12 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_jvm_dtests_vnode:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_stress:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
@@@ -637,28 -791,123 +602,19 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=stress-test-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utests/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -703,170 -949,157 +659,170 @@@
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
- j8_cqlsh_dtests_py3:
- j8_utests_compression_repeat:
++ j8_jvm_dtests_vnode:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
-- parallelism: 100
++ parallelism: 5
steps:
- attach_workspace:
at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
++ name: Determine distributed Tests to Run
+ command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
- run:
- name: Configure virtualenv and python Dependencies
+ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
- run:
- name: Run dtests (j8_without_vnodes)
- name: Repeatedly run new or modifed JUnit tests
-- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist)
command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
+ set -x
export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.6
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
--
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
--
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-compression
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
++ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh_dtests_py38:
++ j8_utests_compression_repeat:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Log Environment Information
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run dtests (j11_without_vnodes)
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -905,12 -1136,10 +861,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j11_repeated_ant_test:
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_unit_tests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: xlarge
@@@ -943,123 -1191,28 +917,28 @@@
which java
java -version
- run:
- name: Run repeated JUnit test
- no_output_timeout: 15m
+ name: Run Unit Tests (testclasslist)
command: |
- if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
- echo "Repeated utest class name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
- echo "Repeated utest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
- echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
-
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- target=${REPEATED_ANT_TEST_TARGET}
- class_path=${REPEATED_ANT_TEST_CLASS}
- class_name="${class_path##*.}"
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" || \
- $target == "test-simulator-dtest" ]]; then
- name="-Dtest.name=$class_name"
- else
- name="-Dtest.name=$class_path"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
- methods=""
- else
- methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
- fi
-
- # Prepare the JVM dtests vnodes argument, which is optional
- vnodes_args=""
- if ${REPEATED_ANT_TEST_VNODES}; then
- vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
- fi
-
- # Run the test target as many times as requested collecting the exit code,
- # stopping the iteration only if stop_on_failure is set.
- exit_code="$?"
- for i in $(seq -w 1 $count); do
-
- echo "Running test iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utest/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
-
- # move the XML output files
- source=build/test/output
- dest=/tmp/results/repeated_utest/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs
- dest=/tmp/results/repeated_utest/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
-
- (exit ${exit_code})
- fi
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
fi
- ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
- store_test_results:
- path: /tmp/results/repeated_utest/output
- - store_artifacts:
- path: /tmp/results/repeated_utest/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utest/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utest/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1363,28 -1488,123 +1223,19 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
- test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
- fi
- ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-compression
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-compression $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utests/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1481,24 -1679,116 +1313,123 @@@
which java
java -version
- run:
- name: Run Simulator Tests
+ name: Run repeated JUnit test
+ no_output_timeout: 15m
command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ if [ "${REPEATED_ANT_TEST_CLASS}" == "<nil>" ]; then
+ echo "Repeated utest class name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" == "<nil>" ]; then
+ echo "Repeated utest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_ANT_TEST_COUNT}" -le 0 ]; then
+ echo "Repeated utest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_ANT_TEST_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_ANT_TEST_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_ANT_TEST_TARGET} ${REPEATED_ANT_TEST_CLASS} ${REPEATED_ANT_TEST_METHODS} ${REPEATED_ANT_TEST_COUNT} times"
+
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+
+ target=${REPEATED_ANT_TEST_TARGET}
+ class_path=${REPEATED_ANT_TEST_CLASS}
+ class_name="${class_path##*.}"
+
+ # Prepare the -Dtest.name argument.
+ # It can be the fully qualified class name or the short class name, depending on the target.
+ if [[ $target == "test" || \
+ $target == "test-cdc" || \
+ $target == "test-compression" || \
+ $target == "test-system-keyspace-directory" || \
+ $target == "fqltool-test" || \
+ $target == "long-test" || \
- $target == "stress-test" ]]; then
++ $target == "stress-test" || \
++ $target == "test-simulator-dtest" ]]; then
+ name="-Dtest.name=$class_name"
+ else
+ name="-Dtest.name=$class_path"
+ fi
+
+ # Prepare the -Dtest.methods argument, which is optional
+ if [ "${REPEATED_ANT_TEST_METHODS}" == "<nil>" ]; then
+ methods=""
+ else
+ methods="-Dtest.methods=${REPEATED_ANT_TEST_METHODS}"
+ fi
+
++ # Prepare the JVM dtests vnodes argument, which is optional
++ vnodes_args=""
++ if ${REPEATED_ANT_TEST_VNODES}; then
++ vnodes_args="-Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16'"
++ fi
++
+ # Run the test target as many times as requested collecting the exit code,
+ # stopping the iteration only if stop_on_failure is set.
+ exit_code="$?"
+ for i in $(seq -w 1 $count); do
+
+ echo "Running test iteration $i of $count"
+
+ # run the test
+ status="passes"
- if !( set -o pipefail && ant $target $name $methods -Dno-build-test=true | tee stdout.txt ); then
++ if !( set -o pipefail && ant $target $name $methods $vnodes_args -Dno-build-test=true | tee stdout.txt ); then
+ status="fails"
+ exit_code=1
+ fi
+
+ # move the stdout output file
+ dest=/tmp/results/repeated_utest/stdout/${status}/${i}
+ mkdir -p $dest
+ mv stdout.txt $dest/${REPEATED_ANT_TEST_TARGET}-${REPEATED_ANT_TEST_CLASS}.txt
+
+ # move the XML output files
+ source=build/test/output
+ dest=/tmp/results/repeated_utest/output/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # move the log files
+ source=build/test/logs
+ dest=/tmp/results/repeated_utest/logs/${status}/${i}
+ mkdir -p $dest
+ if [[ -d $source && -n "$(ls $source)" ]]; then
+ mv $source/* $dest/
+ fi
+
+ # maybe stop iterations on test failure
+ if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
+ break
+ fi
+ done
+
+ (exit ${exit_code})
+ fi
fi
- ant test-simulator-dtest
- no_output_timeout: 30m
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results/repeated_utest/output
- store_artifacts:
- path: /tmp/cassandra/build/test/output
+ path: /tmp/results/repeated_utest/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utest/output
destination: junitxml
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
+ path: /tmp/results/repeated_utest/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1538,13 -1826,13 +1469,14 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- utests_long_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_system_keyspace_directory_repeat:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
@@@ -1577,18 -1865,1735 +1509,1489 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_LONG_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_LONG_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and autom [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-system-keyspace-directory
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
path: /tmp/results/repeated_utests/stdout
destination: stdout
- store_artifacts:
- path: /tmp/results/repeated_utests/output
- destination: junitxml
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py3_vnode:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip- [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_cqlsh_dtests_py3:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j11_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource- [...]
+ - run:
+ name: Run dtests (j11_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
+ export CQLSH_PYTHON=/usr/bin/python3.6
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j11_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j11_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_cdc:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-cdc)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_utests_fqltool:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (fqltool-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant fqltool-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_utests_system_keyspace_directory:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine unit Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist-system-keyspace-directory)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_utests_compression:
++ j11_jvm_dtests_vnode:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 100
++ parallelism: 5
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Determine unit Tests to Run
++ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently executed on every circle container
+ # so the goal here is to get the circleci script to return the tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
- set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of containers we have
+ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
- set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" | grep -v upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Run Unit Tests (testclasslist-compression)
++ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_utests_long:
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_simulator_dtests:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: xlarge
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Run Unit Tests (long-test)
++ name: Determine unit Tests to Run
+ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Run Simulator Tests
++ command: |
++ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
- ant long-test
- no_output_timeout: 15m
++ ant test-simulator-dtest
++ no_output_timeout: 30m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_unit_tests_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_utests_compression:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
++ - run:
++ name: Determine unit Tests to Run
++ command: |
++ # reminder: this code (along with all the steps) is independently executed on every circle container
++ # so the goal here is to get the circleci script to return the tests *this* container will run
++ # which we do via the `circleci` cli tool.
++
++ rm -fr ~/cassandra-dtest/upgrade_tests
++ echo "***java tests***"
++
++ # get all of our unit test filenames
++ set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
++
++ # split up the unit tests into groups based on the number of containers we have
++ set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
++ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
++ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
++ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
++ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=testsome
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
++ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
++ if [ -z "$test_timeout" ]; then
++ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant testsome $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ no_output_timeout: 15m
+ - store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
++ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
- path: /tmp/results/repeated_utests/output
++ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
- path: /tmp/results/repeated_utests/logs
++ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_cqlsh-dtests-py2-no-vnodes:
++ j11_utests_long:
+ docker:
- - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
- parallelism: 100
++ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Run Unit Tests (long-test)
+ command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
- - run:
- name: Run dtests (j8_without_vnodes)
++ time mv ~/cassandra /tmp
++ cd /tmp/cassandra
++ if [ -d ~/dtest_jars ]; then
++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
++ fi
++ ant long-test
+ no_output_timeout: 15m
++ - store_test_results:
++ path: /tmp/cassandra/build/test/output/
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/cassandra/build/test/logs
++ destination: logs
++ environment:
++ - ANT_HOME: /usr/share/ant
++ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - LANG: en_US.UTF-8
++ - KEEP_TEST_DIR: true
++ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
++ - PYTHONIOENCODING: utf-8
++ - PYTHONUNBUFFERED: true
++ - CASS_DRIVER_NO_EXTENSIONS: true
++ - CASS_DRIVER_NO_CYTHON: true
++ - CASSANDRA_SKIP_SYNC: true
++ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
++ - DTEST_BRANCH: trunk
++ - CCM_MAX_HEAP_SIZE: 1024M
++ - CCM_HEAP_NEWSIZE: 256M
++ - REPEATED_TESTS_STOP_ON_FAILURE: false
++ - REPEATED_UTESTS: null
++ - REPEATED_UTESTS_COUNT: 500
++ - REPEATED_UTESTS_FQLTOOL: null
++ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
++ - REPEATED_UTESTS_LONG: null
++ - REPEATED_UTESTS_LONG_COUNT: 100
++ - REPEATED_UTESTS_STRESS: null
++ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
++ - REPEATED_JVM_DTESTS: null
++ - REPEATED_JVM_DTESTS_COUNT: 500
++ - REPEATED_JVM_UPGRADE_DTESTS: null
++ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
++ - REPEATED_DTESTS: null
++ - REPEATED_DTESTS_COUNT: 500
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
++ - CASSANDRA_USE_JDK11: true
++ j8_unit_tests_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
++ resource_class: xlarge
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 100
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
+ command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
+ java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
- path: /tmp/results
++ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
++ path: /tmp/results/repeated_utests/stdout
++ destination: stdout
+ - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
++ path: /tmp/results/repeated_utests/output
++ destination: junitxml
++ - store_artifacts:
++ path: /tmp/results/repeated_utests/logs
++ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j11_utests_stress:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Run Unit Tests (stress-test)
+ command: |
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ ant stress-test
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_upgrade_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if true; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_cqlsh-dtests-py2-with-vnodes:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j11_with_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip [...]
- - run:
- name: Run dtests (j11_with_vnodes)
- no_output_timeout: 15m
- command: |
- echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
- export CQLSH_PYTHON=/usr/bin/python2.7
- fi
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=16 --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- - store_test_results:
- path: /tmp/results
- - store_artifacts:
- path: /tmp/dtest
- destination: dtest_j11_with_vnodes
- - store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j11_with_vnodes_logs
- environment:
- - ANT_HOME: /usr/share/ant
- - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - LANG: en_US.UTF-8
- - KEEP_TEST_DIR: true
- - DEFAULT_DIR: /home/cassandra/cassandra-dtest
- - PYTHONIOENCODING: utf-8
- - PYTHONUNBUFFERED: true
- - CASS_DRIVER_NO_EXTENSIONS: true
- - CASS_DRIVER_NO_CYTHON: true
- - CASSANDRA_SKIP_SYNC: true
- - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: trunk
- - CCM_MAX_HEAP_SIZE: 1024M
- - CCM_HEAP_NEWSIZE: 256M
- - REPEATED_TESTS_STOP_ON_FAILURE: false
- - REPEATED_UTESTS: null
- - REPEATED_UTESTS_COUNT: 500
- - REPEATED_UTESTS_FQLTOOL: null
- - REPEATED_UTESTS_FQLTOOL_COUNT: 500
- - REPEATED_UTESTS_LONG: null
- - REPEATED_UTESTS_LONG_COUNT: 100
- - REPEATED_UTESTS_STRESS: null
- - REPEATED_UTESTS_STRESS_COUNT: 500
- - REPEATED_JVM_DTESTS: null
- - REPEATED_JVM_DTESTS_COUNT: 500
- - REPEATED_JVM_UPGRADE_DTESTS: null
- - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
- - REPEATED_DTESTS: null
- - REPEATED_DTESTS_COUNT: 500
- - REPEATED_UPGRADE_DTESTS: null
- - REPEATED_UPGRADE_DTESTS_COUNT: 25
- - REPEATED_ANT_TEST_TARGET: testsome
- - REPEATED_ANT_TEST_CLASS: null
- - REPEATED_ANT_TEST_METHODS: null
- - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j11_utests_cdc_repeat:
- docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
- resource_class: xlarge
- working_directory: ~/
- shell: /bin/bash -eo pipefail -l
- parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-cdc
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-cdc $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ - REPEATED_UPGRADE_DTESTS: null
++ - REPEATED_UPGRADE_DTESTS_COUNT: 25
++ - REPEATED_ANT_TEST_TARGET: testsome
++ - REPEATED_ANT_TEST_CLASS: null
++ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
++ - REPEATED_ANT_TEST_COUNT: 500
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j11_utests_cdc_repeat:
++ docker:
++ - image: apache/cassandra-testing-ubuntu2004-java11:latest
++ resource_class: xlarge
++ working_directory: ~/
++ shell: /bin/bash -eo pipefail -l
++ parallelism: 100
++ steps:
++ - attach_workspace:
++ at: /home/cassandra
++ - run:
++ name: Log Environment Information
++ command: |
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
++ - run:
++ name: Repeatedly run new or modifed JUnit tests
++ no_output_timeout: 15m
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
++ - REPEATED_SIMULATOR_DTESTS: null
++ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
++ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j11_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Run repeated Python dtest
+ no_output_timeout: 15m
+ command: |
+ if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
+ echo "Repeated dtest name hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
+ echo "Repeated dtest count hasn't been defined, exiting without running any test"
+ elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
+ echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
+ else
+
+ # Calculate the number of test iterations to be run by the current parallel runner.
+ # Since we are running the same test multiple times there is no need to use `circleci tests split`.
+ count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
+ if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
+ count=$((count+1))
+ fi
+
+ if (($count <= 0)); then
+ echo "No tests to run in this runner"
+ else
+ echo "Running ${REPEATED_DTESTS} $count times"
+
+ source ~/env3.6/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+
+ tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
+
+ stop_on_failure_arg=""
+ if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
+ stop_on_failure_arg="-x"
+ fi
+
+ vnodes_args=""
+ if false; then
+ vnodes_args="--use-vnodes --num-tokens=16"
+ fi
+
+ upgrade_arg=""
+ if false; then
+ upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
+ fi
+
+ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
+ set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
+ fi
+ fi
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -1627,11 -3630,11 +3030,12 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_unit_tests_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_utests_fqltool_repeat:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
@@@ -1666,150 -3669,122 +3070,107 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and [...]
+ - store_test_results:
+ path: /tmp/results/repeated_utests/output
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j8_upgrade_dtests_repeat:
++ j8_jvm_dtests_vnode_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
++ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- set -x
-- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_FQLTOOL_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_FQLTOOL_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_FQLTOOL} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=fqltool-test
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant fqltool-test $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ echo '*** id ***'
++ id
++ echo '*** cat /proc/cpuinfo ***'
++ cat /proc/cpuinfo
++ echo '*** free -m ***'
++ free -m
++ echo '*** df -m ***'
++ df -m
++ echo '*** ifconfig -a ***'
++ ifconfig -a
++ echo '*** uname -a ***'
++ uname -a
++ echo '*** mount ***'
++ mount
++ echo '*** env ***'
++ env
++ echo '*** java ***'
++ which java
++ java -version
+ - run:
- name: Run repeated Python dtest
++ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: |
- if [ "${REPEATED_UPGRADE_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_UPGRADE_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_UPGRADE_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_UPGRADE_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
-
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
-
- upgrade_arg=""
- if true; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
-
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -1885,19 -3876,28 +3265,28 @@@
which java
java -version
- run:
- name: Repeatedly run new or modifed JUnit tests
+ name: Run Unit Tests (testclasslist-compression)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}')
+ fi
- ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/results/repeated_utests/output
- - store_artifacts:
- path: /tmp/results/repeated_utests/stdout
- destination: stdout
+ path: /tmp/cassandra/build/test/output/
- store_artifacts:
- path: /tmp/results/repeated_utests/output
+ path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
- path: /tmp/results/repeated_utests/logs
+ path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
@@@ -1937,212 -3935,71 +3326,161 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- j11_dtests_repeat:
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
+ j8_cqlsh_dtests_py38:
docker:
- - image: apache/cassandra-testing-ubuntu2004-java11:latest
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 100
- steps:
- - attach_workspace:
- at: /home/cassandra
- - run:
- name: Log Environment Information
- command: |
- echo '*** id ***'
- id
- echo '*** cat /proc/cpuinfo ***'
- cat /proc/cpuinfo
- echo '*** free -m ***'
- free -m
- echo '*** df -m ***'
- df -m
- echo '*** ifconfig -a ***'
- ifconfig -a
- echo '*** uname -a ***'
- uname -a
- echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
- command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Run repeated Python dtest
- no_output_timeout: 15m
- command: |
- if [ "${REPEATED_DTESTS}" == "<nil>" ]; then
- echo "Repeated dtest name hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" == "<nil>" ]; then
- echo "Repeated dtest count hasn't been defined, exiting without running any test"
- elif [ "${REPEATED_DTESTS_COUNT}" -le 0 ]; then
- echo "Repeated dtest count is lesser or equals than zero, exiting without running any test"
- else
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- # Since we are running the same test multiple times there is no need to use `circleci tests split`.
- count=$((${REPEATED_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- if (($count <= 0)); then
- echo "No tests to run in this runner"
- else
- echo "Running ${REPEATED_DTESTS} $count times"
-
- source ~/env3.6/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
-
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
-
- tests_arg=$(echo ${REPEATED_DTESTS} | sed -e "s/,/ /g")
-
- stop_on_failure_arg=""
- if ${REPEATED_TESTS_STOP_ON_FAILURE}; then
- stop_on_failure_arg="-x"
- fi
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
+ source ~/env3.8/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 uninstall -y cqlsh
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ - run:
- name: Run dtests (j8_without_vnodes)
- no_output_timeout: 15m
++ name: Run dtests (j8_without_vnodes)
++ no_output_timeout: 15m
++ command: |
++ echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
++ cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
+
- vnodes_args=""
- if false; then
- vnodes_args="--use-vnodes --num-tokens=16"
- fi
++ source ~/env3.8/bin/activate
++ export PATH=$JAVA_HOME/bin:$PATH
++ if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
++ export CQLSH_PYTHON=/usr/bin/python3.8
++ fi
+
- upgrade_arg=""
- if false; then
- upgrade_arg="--execute-upgrade-tests --upgrade-target-version-only --upgrade-version-selection all"
- fi
++ java -version
++ cd ~/cassandra-dtest
++ mkdir -p /tmp/dtest
+
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- set -o pipefail && cd ~/cassandra-dtest && pytest $vnodes_args --count=$count $stop_on_failure_arg $upgrade_arg --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $tests_arg | tee /tmp/dtest/stdout.txt
- fi
- fi
++ echo "env: $(env)"
++ echo "** done env"
++ mkdir -p /tmp/results/dtests
++ # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
++ export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
++ set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
- destination: dtest
++ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
- destination: dtest_logs
++ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - ANT_HOME: /usr/share/ant
+ - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: https://github.com/apache/cassandra-dtest.git
+ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - REPEATED_TESTS_STOP_ON_FAILURE: false
+ - REPEATED_UTESTS: null
+ - REPEATED_UTESTS_COUNT: 500
+ - REPEATED_UTESTS_FQLTOOL: null
+ - REPEATED_UTESTS_FQLTOOL_COUNT: 500
+ - REPEATED_UTESTS_LONG: null
+ - REPEATED_UTESTS_LONG_COUNT: 100
+ - REPEATED_UTESTS_STRESS: null
+ - REPEATED_UTESTS_STRESS_COUNT: 500
+ - REPEATED_SIMULATOR_DTESTS: null
+ - REPEATED_SIMULATOR_DTESTS_COUNT: 500
+ - REPEATED_JVM_DTESTS: null
+ - REPEATED_JVM_DTESTS_COUNT: 500
+ - REPEATED_JVM_UPGRADE_DTESTS: null
+ - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500
+ - REPEATED_DTESTS: null
+ - REPEATED_DTESTS_COUNT: 500
+ - REPEATED_UPGRADE_DTESTS: null
+ - REPEATED_UPGRADE_DTESTS_COUNT: 25
+ - REPEATED_ANT_TEST_TARGET: testsome
+ - REPEATED_ANT_TEST_CLASS: null
+ - REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
+ - REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- - CASSANDRA_USE_JDK11: true
- j8_jvm_dtests_vnode_repeat:
++ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
++ j8_simulator_dtests_repeat:
+ docker:
+ - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Log Environment Information
command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- fi
-
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
-
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
+ - run:
+ name: Repeatedly run new or modifed JUnit tests
+ no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_JVM_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automat [...]
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -2196,56 -4050,146 +3534,42 @@@
- attach_workspace:
at: /home/cassandra
- run:
- name: Clone Cassandra dtest Repository (via git)
- command: |
- git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
- - run:
- name: Configure virtualenv and python Dependencies
+ name: Log Environment Information
command: |
- # note, this should be super quick as all dependencies should be pre-installed in the docker image
- # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
- # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
- # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
- source ~/env3.8/bin/activate
- export PATH=$JAVA_HOME/bin:$PATH
- pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt
- pip3 uninstall -y cqlsh
- pip3 freeze
- - run:
- name: Determine Tests to Run (j8_without_vnodes)
- no_output_timeout: 5m
- command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.8/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...]
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
- run:
- name: Run dtests (j8_without_vnodes)
+ name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
-- command: |
- echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
- cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
-
- source ~/env3.8/bin/activate
- set -x
-- export PATH=$JAVA_HOME/bin:$PATH
- if [ -n 'CQLSH_PYTHON=/usr/bin/python3.8' ]; then
- export CQLSH_PYTHON=/usr/bin/python3.8
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
-- fi
--
- java -version
- cd ~/cassandra-dtest
- mkdir -p /tmp/dtest
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
--
- echo "env: $(env)"
- echo "** done env"
- mkdir -p /tmp/results/dtests
- # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
- export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
- set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-system-keyspace-directory
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant test-system-keyspace-directory $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...]
- store_test_results:
- path: /tmp/results
+ path: /tmp/results/repeated_utests/output
- store_artifacts:
- path: /tmp/dtest
- destination: dtest_j8_without_vnodes
+ path: /tmp/results/repeated_utests/stdout
+ destination: stdout
- store_artifacts:
- path: ~/cassandra-dtest/logs
- destination: dtest_j8_without_vnodes_logs
+ path: /tmp/results/repeated_utests/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/results/repeated_utests/logs
+ destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
@@@ -2323,7 -4264,111 +3647,7 @@@
- run:
name: Repeatedly run new or modifed JUnit tests
no_output_timeout: 15m
- command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_SIMULATOR_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_SIMULATOR_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests [...]
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_UTESTS_STRESS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=stress-test-some
- testtag=""
- if [[ $target == "test-cdc" ]]; then
- testtag="cdc"
- elif [[ $target == "test-compression" ]]; then
- testtag="compression"
- elif [[ $target == "test-system-keyspace-directory" ]]; then
- testtag="system_keyspace_directory"
- fi
-
- # Run each test class as many times as requested.
- exit_code="$?"
- for test in $tests; do
-
- # Split class and method names from the test name
- if [[ $test =~ "#" ]]; then
- class=${test%"#"*}
- method=${test#*"#"}
- else
- class=$test
- method=""
- fi
-
- # Prepare the -Dtest.name argument.
- # It can be the fully qualified class name or the short class name, depending on the target.
- if [[ $target == "test" || \
- $target == "test-cdc" || \
- $target == "test-compression" || \
- $target == "test-system-keyspace-directory" || \
- $target == "fqltool-test" || \
- $target == "long-test" || \
- $target == "stress-test" ]]; then
- name_arg="-Dtest.name=${class##*.}"
- else
- name_arg="-Dtest.name=$class"
- fi
-
- # Prepare the -Dtest.methods argument, which is optional
- if [[ $method == "" ]]; then
- methods_arg=""
- else
- methods_arg="-Dtest.methods=$method"
- fi
-
- for i in $(seq -w 1 $count); do
- echo "Running test $test, iteration $i of $count"
-
- # run the test
- status="passes"
- if !( set -o pipefail && \
- ant stress-test-some $name_arg $methods_arg -Dno-build-test=true | \
- tee stdout.txt \
- ); then
- status="fails"
- exit_code=1
- fi
-
- # move the stdout output file
- dest=/tmp/results/repeated_utests/stdout/${status}/${i}
- mkdir -p $dest
- mv stdout.txt $dest/${test}.txt
-
- # move the XML output files
- source=build/test/output/${testtag}
- dest=/tmp/results/repeated_utests/output/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # move the log files
- source=build/test/logs/${testtag}
- dest=/tmp/results/repeated_utests/logs/${status}/${i}
- mkdir -p $dest
- if [[ -d $source && -n "$(ls $source)" ]]; then
- mv $source/* $dest/
- fi
-
- # maybe stop iterations on test failure
- if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then
- break
- fi
- done
- done
- (exit ${exit_code})
++ command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_STRESS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_STRESS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and a [...]
- store_test_results:
path: /tmp/results/repeated_utests/output
- store_artifacts:
@@@ -2698,10 -4645,10 +3932,11 @@@
- REPEATED_ANT_TEST_TARGET: testsome
- REPEATED_ANT_TEST_CLASS: null
- REPEATED_ANT_TEST_METHODS: null
+ - REPEATED_ANT_TEST_VNODES: false
- REPEATED_ANT_TEST_COUNT: 500
- - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
+ - CASSANDRA_USE_JDK11: true
j8_upgrade_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@@@ -2812,16 -4756,120 +4047,16 @@@
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
- mount
- echo '*** env ***'
- env
- echo '*** java ***'
- which java
- java -version
- - run:
- name: Repeatedly run new or modifed JUnit tests
- no_output_timeout: 15m
- command: |
- set -x
- export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- if [ -d ~/dtest_jars ]; then
- cp ~/dtest_jars/dtest* /tmp/cassandra/build/
- fi
-
- # Calculate the number of test iterations to be run by the current parallel runner.
- count=$((${REPEATED_JVM_UPGRADE_DTESTS_COUNT} / CIRCLE_NODE_TOTAL))
- if (($CIRCLE_NODE_INDEX < (${REPEATED_JVM_UPGRADE_DTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then
- count=$((count+1))
- fi
-
- # Put manually specified tests and automatically detected tests together, removing duplicates
- tests=$(echo ${REPEATED_JVM_UPGRADE_DTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u)
- echo "Tests to be repeated: ${tests}"
-
- # Prepare the testtag for the target, used by the test macro in build.xml to group the output files
- target=test-jvm-dtest-some
- testtag=""
... 12474 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org